From 0b4fadd7713dc9988da365fddfb9951364219a2a Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 14 Oct 2019 14:09:34 +0100 Subject: [PATCH 001/257] Updated CHANGELOG. (#1645) Signed-off-by: Bartek Plotka --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b97e32c2e..dc2db529d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,9 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Fixed - [#1632](https://github.com/thanos-io/thanos/issues/1632) Removes the duplicated external labels detection on Thanos Querier; warning only; Made Store Gateway compatible with older Querier versions. - NOTE: `thanos_store_nodes_grpc_connections` metric is now per `external_labels` and `store_type`. It is a recommended metric for Querier storeAPIs. `thanos_store_node_info` is marked as obsolete and will be removed in next release. - NOTE2: Store Gateway is not advertising artificial: `"@thanos_compatibility_store_type=store"` label. This is to have current Store Gateway compatible with Querier pre v0.8.0. -This label can be disabled by hidden `debug.advertise-compatibility-label=false` flag on Store Gateway. + * NOTE: `thanos_store_nodes_grpc_connections` metric is now per `external_labels` and `store_type`. It is a recommended metric for Querier storeAPIs. `thanos_store_node_info` is marked as obsolete and will be removed in next release. + * NOTE2: Store Gateway is now advertising artificial: `"@thanos_compatibility_store_type=store"` label. This is to have the current Store Gateway compatible with Querier pre v0.8.0. + This label can be disabled by hidden `debug.advertise-compatibility-label=false` flag on Store Gateway. ## [v0.8.0](https://github.com/thanos-io/thanos/releases/tag/v0.8.0) - 2019.10.10 From b9101c0f411f37af20eb6c1d6bc2e1c340ae79ac Mon Sep 17 00:00:00 2001 From: Philip Panyukov Date: Mon, 14 Oct 2019 17:26:57 +0100 Subject: [PATCH 002/257] nit: ensure build works when GOPATH is not set (#1647) Signed-off-by: Philip Panyukov --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 05773257ce..b890fb02f6 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,10 @@ DOCKER_IMAGE_REPO ?= quay.io/thanos/thanos DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))-$(shell date +%Y-%m-%d)-$(shell git rev-parse --short HEAD) DOCKER_CI_TAG ?= test +# Ensure everything works even if GOPATH is not set, which is often the case. +# The `go env GOPATH` will work for all cases for Go 1.8+. +GOPATH ?= $(shell go env GOPATH) + TMP_GOPATH ?= /tmp/thanos-go GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin GO111MODULE ?= on From 83fcad51c731a1bdd511c86078d68a428c458b7b Mon Sep 17 00:00:00 2001 From: Ivan Kiselev Date: Tue, 15 Oct 2019 11:57:17 +0200 Subject: [PATCH 003/257] doc on how to disable default partial response behaviour (#1650) * doc on how to disable default partial response behaviour partial request disabling flag documented Signed-off-by: Ivan Kiselev * extra chars in the end of the line Signed-off-by: Ivan Kiselev * update command line params for query Signed-off-by: Ivan Kiselev * Dot in the end Signed-off-by: Ivan Kiselev * goddamn it whitespace Signed-off-by: Ivan Kiselev * goddamnit whitespaces Signed-off-by: Ivan Kiselev * final dot Signed-off-by: Ivan Kiselev --- cmd/thanos/query.go | 2 +- docs/components/query.md | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 827079509c..9202ac6300 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -94,7 +94,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { enableAutodownsampling := cmd.Flag("query.auto-downsampling", "Enable automatic adjustment (step / 5) to what source of data should be used in store gateways if no max_source_resolution param is specified."). Default("false").Bool() - enablePartialResponse := cmd.Flag("query.partial-response", "Enable partial response for queries if no partial_response param is specified."). + enablePartialResponse := cmd.Flag("query.partial-response", "Enable partial response for queries if no partial_response param is specified. --no-query.partial-response for disabling."). Default("true").Bool() defaultEvaluationInterval := modelDuration(cmd.Flag("query.default-evaluation-interval", "Set default evaluation interval for sub queries.").Default("1m")) diff --git a/docs/components/query.md b/docs/components/query.md index 6b6e9353f6..c10c03b328 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -22,10 +22,10 @@ $ thanos query \ ``` ## Querier use cases, why do I need this component? -Thanos Querier essentially allows to aggregate and optionally deduplicate multiple metrics backends under single Prometheus Query endpoint. +Thanos Querier essentially allows to aggregate and optionally deduplicate multiple metrics backends under single Prometheus Query endpoint. ### Global View - + Since for Querier "a backend" is anything that implements gRPC StoreAPI we can aggregate data from any number of the different storages like: * Prometheus (see [Sidecar](sidecar.md)) @@ -50,8 +50,8 @@ even if those clusters runs multiple Prometheus servers each. Querier will know Prometheus is stateful and does not allow replicating its database. This means that increasing high availability by running multiple Prometheus replicas is not very easy to use. Simple loadbalancing will not work as for example after some crash, replica might be up but querying such replica will result in small gap during the period it was down. You have a second replica that maybe was up, but it could be down in other moment (e.g rolling restart), so load balancing on top of those is not working well. - -Thanos Querier instead pulls the data from both replicas, and deduplicate those signals, filling the gaps if any, transparently to the Querier consumer. + +Thanos Querier instead pulls the data from both replicas, and deduplicate those signals, filling the gaps if any, transparently to the Querier consumer. ## Metric Query Flow Overview @@ -336,6 +336,7 @@ Flags: if no max_source_resolution param is specified. --query.partial-response Enable partial response for queries if no partial_response param is specified. + --no-query.partial-response for disabling. --query.default-evaluation-interval=1m Set default evaluation interval for sub queries. From f529f2b112f8e62bd883b4b4c445fc2f990e4b6e Mon Sep 17 00:00:00 2001 From: Bobby Wertman Date: Tue, 15 Oct 2019 07:12:03 -0400 Subject: [PATCH 004/257] Remove duplicated sentence in docs (#1638) Signed-off-by: Bobby Wertman --- docs/quick-tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quick-tutorial.md b/docs/quick-tutorial.md index b7867ef0b7..437337ed38 100644 --- a/docs/quick-tutorial.md +++ b/docs/quick-tutorial.md @@ -214,7 +214,7 @@ thanos compact \ --http-address 0.0.0.0:19191 # HTTP endpoint for collecting metrics on the Compactor ``` -The compactor is not in the critical path of querying or data backup. It can either be run as a periodic batch job or be left running to always compact data as soon as possible. It is recommended to provide 100-300GB of local disk space for data processing. It is recommended to provide 100-300GB of local disk space for data processing. +The compactor is not in the critical path of querying or data backup. It can either be run as a periodic batch job or be left running to always compact data as soon as possible. It is recommended to provide 100-300GB of local disk space for data processing. _NOTE: The compactor must be run as a **singleton** and must not run when manually modifying data in the bucket._ From 5d107d1c3cdc2a42471e9e4b40c60fd676fdcb4a Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 15 Oct 2019 13:50:29 +0100 Subject: [PATCH 005/257] Added CI testing against S3 bucket. (#1648) * Added CI testing against S3 bucket. * Again, only for master/release builds and PRs from person with write-access to Thanos. Signed-off-by: Bartek Plotka * Fixed multipart upload for S3 for smaller objects. Signed-off-by: Bartek Plotka --- .circleci/config.yml | 5 +-- Makefile | 16 ++++---- cmd/thanos/bucket.go | 10 ++++- docs/storage.md | 16 ++++---- .../objtesting/acceptance_e2e_test.go | 3 +- pkg/objstore/objtesting/foreach.go | 7 ++-- pkg/objstore/s3/s3.go | 41 ++++++++++--------- scripts/cfggen/main.go | 2 +- 8 files changed, 53 insertions(+), 47 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4a6fdbc90..29845cfcfb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: command: | if ! [ -z ${GCP_PROJECT} ]; then echo $GOOGLE_APPLICATION_CREDENTIALS_CONTENT > $GOOGLE_APPLICATION_CREDENTIALS - echo "Awesome! GCS integration tests are enabled." + echo "Awesome! GCS and S3 AWS integration tests are enabled." fi - run: make deps - run: make lint @@ -34,14 +34,13 @@ jobs: - run: make format - run: name: "Run all tests" - # TODO(bwplotka): Setup some S3 tests for CI. # taskset sets CPU affinity to 2 (current CPU limit). command: | if [ -z ${GCP_PROJECT} ]; then taskset 2 make test-local exit fi - taskset 2 make test-only-gcs + taskset 2 make test-ci # Cross build is needed for publish_release but needs to be done outside of docker. cross_build: diff --git a/Makefile b/Makefile index b890fb02f6..0f81720236 100644 --- a/Makefile +++ b/Makefile @@ -206,13 +206,11 @@ test: check-git install-deps @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets" @go test $(shell go list ./... | grep -v /vendor/); -.PHONY: test-only-gcs -test-only-gcs: export THANOS_SKIP_S3_AWS_TESTS = true -test-only-gcs: export THANOS_SKIP_AZURE_TESTS = true -test-only-gcs: export THANOS_SKIP_SWIFT_TESTS = true -test-only-gcs: export THANOS_SKIP_TENCENT_COS_TESTS = true -test-only-gcs: - @echo ">> Skipping S3 tests" +.PHONY: test-ci +test-ci: export THANOS_SKIP_AZURE_TESTS = true +test-ci: export THANOS_SKIP_SWIFT_TESTS = true +test-ci: export THANOS_SKIP_TENCENT_COS_TESTS = true +test-ci: @echo ">> Skipping AZURE tests" @echo ">> Skipping SWIFT tests" @echo ">> Skipping TENCENT tests" @@ -220,9 +218,11 @@ test-only-gcs: .PHONY: test-local test-local: export THANOS_SKIP_GCS_TESTS = true +test-local: export THANOS_SKIP_S3_AWS_TESTS = true test-local: @echo ">> Skipping GCE tests" - $(MAKE) test-only-gcs + @echo ">> Skipping S3 tests" + $(MAKE) test-ci # install-deps installs dependencies for e2e tetss. # It installs supported versions of Prometheus and alertmanager to test against in e2e. diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 05f98ecd71..4bc7085e0b 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -178,6 +178,7 @@ func registerBucketLs(m map[string]setupFunc, root *kingpin.CmdClause, name stri var ( format = *output + objects = 0 printBlock func(id ulid.ULID) error ) @@ -234,13 +235,18 @@ func registerBucketLs(m map[string]setupFunc, root *kingpin.CmdClause, name stri } } - return bkt.Iter(ctx, "", func(name string) error { + if err := bkt.Iter(ctx, "", func(name string) error { id, ok := block.IsBlockDir(name) if !ok { return nil } + objects++ return printBlock(id) - }) + }); err != nil { + return errors.Wrap(err, "iter") + } + level.Info(logger).Log("msg", "ls done", "objects", objects) + return nil } } diff --git a/docs/storage.md b/docs/storage.md index ab3a3d37fb..20d815a439 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -55,11 +55,11 @@ Current object storage client implementations: | Provider | Maturity | Auto-tested on CI | Maintainers | |----------------------|-------------------|-----------|---------------| -| [Google Cloud Storage](#gcs) | Stable (production usage) | yes | @bwplotka | -| [AWS/S3](#s3) | Stable (production usage) | yes | @bwplotka | -| [Azure Storage Account](#azure) | Stable (production usage) | yes | @vglafirov | -| [OpenStack Swift](#openstack-swift) | Beta (working PoCs, testing usage) | no | @sudhi-vm | -| [Tencent COS](#tencent-cos) | Beta (testing usage) | no | @jojohappy | +| [Google Cloud Storage](./storage.md#gcs) | Stable (production usage) | yes | @bwplotka | +| [AWS/S3](./storage.md#s3) | Stable (production usage) | yes | @bwplotka | +| [Azure Storage Account](./storage.md#azure) | Stable (production usage) | yes | @vglafirov | +| [OpenStack Swift](./storage.md#openstack-swift) | Beta (working PoCs, testing usage) | no | @sudhi-vm | +| [Tencent COS](./storage.md#tencent-cos) | Beta (testing usage) | no | @jojohappy | NOTE: Currently Thanos requires strong consistency (write-read) for object store implementation. @@ -85,12 +85,12 @@ config: secret_key: "" put_user_metadata: {} http_config: - idle_conn_timeout: 0s - response_header_timeout: 0s + idle_conn_timeout: 90s + response_header_timeout: 2m insecure_skip_verify: false trace: enable: false - part_size: 0 + part_size: 134217728 ``` At a minimum, you will need to provide a value for the `bucket`, `endpoint`, `access_key`, and `secret_key` keys. The rest of the keys are optional. diff --git a/pkg/objstore/objtesting/acceptance_e2e_test.go b/pkg/objstore/objtesting/acceptance_e2e_test.go index e35cb75c44..54c5894ada 100644 --- a/pkg/objstore/objtesting/acceptance_e2e_test.go +++ b/pkg/objstore/objtesting/acceptance_e2e_test.go @@ -98,7 +98,8 @@ func TestObjStore_AcceptanceTest_e2e(t *testing.T) { testutil.Ok(t, bkt.Delete(ctx, "id1/obj_2.some")) // Delete is expected to fail on non existing object. - testutil.NotOk(t, bkt.Delete(ctx, "id1/obj_2.some")) + // NOTE: Don't rely on this. S3 is not complying with this as GCS is. + // testutil.NotOk(t, bkt.Delete(ctx, "id1/obj_2.some")) // Can we iter over items from id1/ dir and see obj2 being deleted? seen = []string{} diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go index 66dc47b24f..0b5dffcb03 100644 --- a/pkg/objstore/objtesting/foreach.go +++ b/pkg/objstore/objtesting/foreach.go @@ -37,7 +37,7 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) testutil.Ok(t, err) ok := t.Run("gcs", func(t *testing.T) { - // TODO(bplotka): Add leaktest when https://github.com/GoogleCloudPlatform/google-cloud-go/issues/1025 is resolved. + // TODO(bwplotka): Add leaktest when https://github.com/GoogleCloudPlatform/google-cloud-go/issues/1025 is resolved. testFn(t, bkt) }) closeFn() @@ -48,11 +48,10 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) t.Log("THANOS_SKIP_GCS_TESTS envvar present. Skipping test against GCS.") } - // Optional S3 AWS. - // TODO(bwplotka): Prepare environment & CI to run it automatically. + // Optional S3. if _, ok := os.LookupEnv("THANOS_SKIP_S3_AWS_TESTS"); !ok { // TODO(bwplotka): Allow taking location from envvar. - bkt, closeFn, err := s3.NewTestBucket(t, "eu-west-1") + bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2") testutil.Ok(t, err) ok := t.Run("aws s3", func(t *testing.T) { diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index a687c7ba7b..bf33400a20 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -32,9 +32,16 @@ import ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" -// Minimum file size after which an HTTP multipart request should be used to upload objects to storage. -// Set to 128 MiB as in the minio client. -const defaultMinPartSize = 1024 * 1024 * 128 +var DefaultConfig = Config{ + PutUserMetadata: map[string]string{}, + HTTPConfig: HTTPConfig{ + IdleConnTimeout: model.Duration(90 * time.Second), + ResponseHeaderTimeout: model.Duration(2 * time.Minute), + }, + // Minimum file size after which an HTTP multipart request should be used to upload objects to storage. + // Set to 128 MiB as in the minio client. + PartSize: 1024 * 1024 * 128, +} // Config stores the configuration for s3 bucket. type Config struct { @@ -49,7 +56,8 @@ type Config struct { PutUserMetadata map[string]string `yaml:"put_user_metadata"` HTTPConfig HTTPConfig `yaml:"http_config"` TraceConfig TraceConfig `yaml:"trace"` - PartSize uint64 `yaml:"part_size"` + // PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize. + PartSize uint64 `yaml:"part_size"` } type TraceConfig struct { @@ -75,23 +83,11 @@ type Bucket struct { // parseConfig unmarshals a buffer into a Config with default HTTPConfig values. func parseConfig(conf []byte) (Config, error) { - defaultHTTPConfig := HTTPConfig{ - IdleConnTimeout: model.Duration(90 * time.Second), - ResponseHeaderTimeout: model.Duration(2 * time.Minute), - } - config := Config{HTTPConfig: defaultHTTPConfig} + config := DefaultConfig if err := yaml.Unmarshal(conf, &config); err != nil { return Config{}, err } - if config.PutUserMetadata == nil { - config.PutUserMetadata = make(map[string]string) - } - - if config.PartSize == 0 { - config.PartSize = defaultMinPartSize - } - return config, nil } @@ -314,16 +310,21 @@ func (b *Bucket) guessFileSize(name string, r io.Reader) int64 { // Upload the contents of the reader as an object into the bucket. func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. - fileSize := b.guessFileSize(name, r) + size := b.guessFileSize(name, r) + // partSize cannot be larger than object size. + partSize := b.partSize + if size < int64(partSize) { + partSize = 0 + } if _, err := b.client.PutObjectWithContext( ctx, b.name, name, r, - fileSize, + size, minio.PutObjectOptions{ - PartSize: b.partSize, + PartSize: partSize, ServerSideEncryption: b.sse, UserMetadata: b.putUserMetadata, }, diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 8174f5d1a9..4bf93a69c8 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -30,7 +30,7 @@ var ( bucketConfigs = map[client.ObjProvider]interface{}{ client.AZURE: azure.Config{}, client.GCS: gcs.Config{}, - client.S3: s3.Config{}, + client.S3: s3.DefaultConfig, client.SWIFT: swift.SwiftConfig{}, client.COS: cos.Config{}, } From fb0db632da80e7870a56048c38b8241b45388b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Tue, 15 Oct 2019 15:23:05 +0200 Subject: [PATCH 006/257] cmd/thanos/flags: remove unhelpful flags helper (#1652) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit cleans up a TODO in the flags.go file and eliminates the regCommonServerFlags func, which added almost no additional utility beyond calling regGRPCFlags and regHTTPAddrFlag. Signed-off-by: Lucas Servén Marín --- cmd/thanos/flags.go | 17 ----------------- cmd/thanos/query.go | 3 ++- cmd/thanos/rule.go | 3 ++- cmd/thanos/sidecar.go | 3 ++- cmd/thanos/store.go | 3 ++- 5 files changed, 8 insertions(+), 21 deletions(-) diff --git a/cmd/thanos/flags.go b/cmd/thanos/flags.go index 5d8ef8e1b2..f8cd6570cc 100644 --- a/cmd/thanos/flags.go +++ b/cmd/thanos/flags.go @@ -29,23 +29,6 @@ func regGRPCFlags(cmd *kingpin.CmdClause) ( grpcTLSSrvClientCA } -// TODO(povilasv): we don't need this anymore. -func regCommonServerFlags(cmd *kingpin.CmdClause) ( - grpcBindAddr *string, - httpBindAddr *string, - grpcTLSSrvCert *string, - grpcTLSSrvKey *string, - grpcTLSSrvClientCA *string) { - httpBindAddr = regHTTPAddrFlag(cmd) - grpcBindAddr, grpcTLSSrvCert, grpcTLSSrvKey, grpcTLSSrvClientCA = regGRPCFlags(cmd) - - return grpcBindAddr, - httpBindAddr, - grpcTLSSrvCert, - grpcTLSSrvKey, - grpcTLSSrvClientCA -} - func regHTTPAddrFlag(cmd *kingpin.CmdClause) *string { return cmd.Flag("http-address", "Listen host:port for HTTP endpoints.").Default("0.0.0.0:10902").String() } diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 9202ac6300..6821e6fd36 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -47,7 +47,8 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { comp := component.Query cmd := app.Command(comp.String(), "query node exposing PromQL enabled Query API with data retrieved from multiple store nodes") - grpcBindAddr, httpBindAddr, srvCert, srvKey, srvClientCA := regCommonServerFlags(cmd) + httpBindAddr := regHTTPAddrFlag(cmd) + grpcBindAddr, srvCert, srvKey, srvClientCA := regGRPCFlags(cmd) secure := cmd.Flag("grpc-client-tls-secure", "Use TLS when talking to the gRPC server").Default("false").Bool() cert := cmd.Flag("grpc-client-tls-cert", "TLS Certificates to use to identify this client to the server").Default("").String() diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index bf9344d399..b5cc568ebf 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -61,7 +61,8 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { comp := component.Rule cmd := app.Command(comp.String(), "ruler evaluating Prometheus rules against given Query nodes, exposing Store API and storing old blocks in bucket") - grpcBindAddr, httpBindAddr, cert, key, clientCA := regCommonServerFlags(cmd) + httpBindAddr := regHTTPAddrFlag(cmd) + grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) labelStrs := cmd.Flag("label", "Labels to be applied to all generated metrics (repeated). Similar to external labels for Prometheus, used to identify ruler and its blocks as unique source."). PlaceHolder("=\"\"").Strings() diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 9a39d63e8f..403bce843e 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -37,7 +37,8 @@ const waitForExternalLabelsTimeout = 10 * time.Minute func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(component.Sidecar.String(), "sidecar for Prometheus server") - grpcBindAddr, httpBindAddr, cert, key, clientCA := regCommonServerFlags(cmd) + httpBindAddr := regHTTPAddrFlag(cmd) + grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) promURL := cmd.Flag("prometheus.url", "URL at which to reach Prometheus's API. For better performance use local network."). Default("http://localhost:9090").URL() diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index fdb84f1647..139a7c8a74 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -29,7 +29,8 @@ import ( func registerStore(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(component.Store.String(), "store node giving access to blocks in a bucket provider. Now supported GCS, S3, Azure, Swift and Tencent COS.") - grpcBindAddr, httpBindAddr, cert, key, clientCA := regCommonServerFlags(cmd) + httpBindAddr := regHTTPAddrFlag(cmd) + grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() From 48a8fb6e2f6a476bcffa508d6609a19847c695ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Wed, 16 Oct 2019 11:31:44 +0200 Subject: [PATCH 007/257] pkg/receive: remove flushed WAL (#1654) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit ensures that we delete the WAL after it has been flushed to a block. Flushing the WAL simply creates a block but does not remove the WAL directory or its contents. This means that once the DB is re-opened, new samples are added to the same WAL. Flushing the WAL again does not result in blocks with overlapping time ranges because the flushing logic guards against this (https://github.com/prometheus/prometheus/blob/master/tsdb/db.go#L300). Nevertheless, we should delete the WAL after flushing it to ensure that flushed samples are not needlessly re-processed. Also, once multi-TSDB support is added, holding old samples in the WAL could cause problems. Signed-off-by: Lucas Servén Marín --- pkg/receive/tsdb.go | 5 +++ pkg/receive/tsdb_test.go | 80 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 pkg/receive/tsdb_test.go diff --git a/pkg/receive/tsdb.go b/pkg/receive/tsdb.go index b2f504a535..16fdae6e8f 100644 --- a/pkg/receive/tsdb.go +++ b/pkg/receive/tsdb.go @@ -1,6 +1,8 @@ package receive import ( + "os" + "path/filepath" "sync" "github.com/go-kit/kit/log" @@ -86,6 +88,9 @@ func (f *FlushableStorage) Flush() error { if err := ro.FlushWAL(f.Dir()); err != nil { return errors.Wrap(err, "flushing WAL") } + if err := os.RemoveAll(filepath.Join(f.Dir(), "wal")); err != nil { + return errors.Wrap(err, "removing stale WAL") + } if reopen { return errors.Wrap(f.open(), "re-starting storage") } diff --git a/pkg/receive/tsdb_test.go b/pkg/receive/tsdb_test.go new file mode 100644 index 0000000000..cd9ed3651e --- /dev/null +++ b/pkg/receive/tsdb_test.go @@ -0,0 +1,80 @@ +package receive + +import ( + "io/ioutil" + "os" + "testing" + "time" + + "github.com/go-kit/kit/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/storage/tsdb" + "github.com/prometheus/prometheus/tsdb/labels" + + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestFlushableStorage(t *testing.T) { + { + // Ensure that flushing storage does not cause data loss. + // This test: + // * opens a flushable storage; + // * appends values; + // * flushes the storage; and + // * queries the storage to ensure the samples are present. + + dbDir, err := ioutil.TempDir("", "test") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dbDir)) }() + + tsdbCfg := &tsdb.Options{ + RetentionDuration: model.Duration(time.Hour * 24 * 15), + NoLockfile: true, + MinBlockDuration: model.Duration(time.Hour * 2), + MaxBlockDuration: model.Duration(time.Hour * 2), + WALCompression: true, + } + + db := NewFlushableStorage( + dbDir, + log.NewNopLogger(), + prometheus.NewRegistry(), + tsdbCfg, + ) + + testutil.Ok(t, db.Open()) + defer func() { testutil.Ok(t, db.Close()) }() + + // Append data to the WAL. + app := db.Appender() + maxt := 1000 + for i := 0; i < maxt; i++ { + _, err := app.Add(labels.FromStrings("thanos", "flush"), int64(i), 1.0) + testutil.Ok(t, err) + } + testutil.Ok(t, app.Commit()) + + // Flush the WAL. + testutil.Ok(t, db.Flush()) + + querier, err := db.Querier(0, int64(maxt)-1) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, querier.Close()) }() + + // Sum the values. + seriesSet, err := querier.Select(labels.NewEqualMatcher("thanos", "flush")) + testutil.Ok(t, err) + sum := 0.0 + for seriesSet.Next() { + series := seriesSet.At().Iterator() + for series.Next() { + _, v := series.At() + sum += v + } + testutil.Ok(t, series.Err()) + } + testutil.Ok(t, seriesSet.Err()) + testutil.Equals(t, 1000.0, sum) + } +} From c88d3d6a3a217de0e660bec4815150a8ce18d77c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2019 11:33:04 +0100 Subject: [PATCH 008/257] build(deps): bump github.com/miekg/dns from 1.1.19 to 1.1.22 (#1620) Bumps [github.com/miekg/dns](https://github.com/miekg/dns) from 1.1.19 to 1.1.22. - [Release notes](https://github.com/miekg/dns/releases) - [Changelog](https://github.com/miekg/dns/blob/master/Makefile.release) - [Commits](https://github.com/miekg/dns/compare/v1.1.19...v1.1.22) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 808ff87ad1..a04e80ba3d 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/lovoo/gcloud-opentracing v0.3.0 github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect; Pinned for FreeBSD support. github.com/mattn/go-runewidth v0.0.4 // indirect - github.com/miekg/dns v1.1.19 + github.com/miekg/dns v1.1.22 github.com/minio/minio-go/v6 v6.0.27-0.20190529152532-de69c0e465ed github.com/mozillazg/go-cos v0.12.0 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f diff --git a/go.sum b/go.sum index e491effcee..04a7e43588 100644 --- a/go.sum +++ b/go.sum @@ -320,6 +320,8 @@ github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.19 h1:0ymbfaLG1/utH2+BydNiF+dx1jSEmdr/nylOtkGHZZg= github.com/miekg/dns v1.1.19/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc= +github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= github.com/minio/minio-go/v6 v6.0.27-0.20190529152532-de69c0e465ed h1:g3DRJpu22jEjs14fSeJ7Crn9vdreiRsn4RtrEsXH/6A= github.com/minio/minio-go/v6 v6.0.27-0.20190529152532-de69c0e465ed/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y= From f0d3b14fad6476eb2b97f1c50d688aba8f73df1b Mon Sep 17 00:00:00 2001 From: Andras Ferencz-Szabo Date: Fri, 18 Oct 2019 10:05:09 +0100 Subject: [PATCH 009/257] Add a new prometheus.ready_timeout CLI option to the sidecar (#1660) The current timeout while the Prometheus instance is starting up is a constant 10 minutes. As reading the WAL can take a long time we would like to set a custom timeout value, so the Thanos sidecar container is not erroring out every 10 minutes. Signed-off-by: Andras Ferencz-Szabo --- CHANGELOG.md | 4 ++++ cmd/thanos/sidecar.go | 11 +++++++---- docs/components/sidecar.md | 3 +++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc2db529d9..c7c45a703b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased +### Added + +- [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. + ## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14 ### Fixed diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 403bce843e..dfc2b4ac56 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -32,8 +32,6 @@ import ( "gopkg.in/alecthomas/kingpin.v2" ) -const waitForExternalLabelsTimeout = 10 * time.Minute - func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(component.Sidecar.String(), "sidecar for Prometheus server") @@ -43,6 +41,9 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { promURL := cmd.Flag("prometheus.url", "URL at which to reach Prometheus's API. For better performance use local network."). Default("http://localhost:9090").URL() + promReadyTimeout := cmd.Flag("prometheus.ready_timeout", "Maximum time to wait for the Prometheus instance to start up"). + Default("10m").Duration() + dataDir := cmd.Flag("tsdb.path", "Data directory of TSDB."). Default("./data").String() @@ -81,6 +82,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { *clientCA, *httpBindAddr, *promURL, + *promReadyTimeout, *dataDir, objStoreConfig, rl, @@ -102,6 +104,7 @@ func runSidecar( clientCA string, httpBindAddr string, promURL *url.URL, + promReadyTimeout time.Duration, dataDir string, objStoreConfig *extflag.PathOrContent, reloader *reloader.Reloader, @@ -268,7 +271,7 @@ func runSidecar( g.Add(func() error { defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") - extLabelsCtx, cancel := context.WithTimeout(ctx, waitForExternalLabelsTimeout) + extLabelsCtx, cancel := context.WithTimeout(ctx, promReadyTimeout) defer cancel() if err := runutil.Retry(2*time.Second, extLabelsCtx.Done(), func() error { @@ -277,7 +280,7 @@ func runSidecar( } return nil }); err != nil { - return errors.Wrapf(err, "aborting as no external labels found after waiting %s", waitForExternalLabelsTimeout) + return errors.Wrapf(err, "aborting as no external labels found after waiting %s", promReadyTimeout) } var s *shipper.Shipper diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 60699f125f..a1d1a92112 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -116,6 +116,9 @@ Flags: --prometheus.url=http://localhost:9090 URL at which to reach Prometheus's API. For better performance use local network. + --prometheus.ready_timeout=10m + Maximum time to wait for the Prometheus + instance to start up --tsdb.path="./data" Data directory of TSDB. --reloader.config-file="" Config file watched by the reloader. --reloader.config-envsubst-file="" From 32149d06b52a4422d8c1ef4b40a26c20e403a6f6 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 18 Oct 2019 18:48:06 +0100 Subject: [PATCH 010/257] Upgraded minio-go dep. (#1658) Signed-off-by: Bartek Plotka --- go.mod | 6 ++++-- go.sum | 19 ++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index a04e80ba3d..a9cd7f1f63 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect; Pinned for FreeBSD support. github.com/mattn/go-runewidth v0.0.4 // indirect github.com/miekg/dns v1.1.22 - github.com/minio/minio-go/v6 v6.0.27-0.20190529152532-de69c0e465ed + github.com/minio/minio-go/v6 v6.0.39 github.com/mozillazg/go-cos v0.12.0 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/oklog/run v1.0.0 @@ -43,9 +43,11 @@ require ( go.elastic.co/apm v1.5.0 go.elastic.co/apm/module/apmot v1.5.0 go.uber.org/automaxprocs v1.2.0 - golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc // indirect + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect + golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect golang.org/x/text v0.3.2 google.golang.org/api v0.11.0 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 diff --git a/go.sum b/go.sum index 04a7e43588..655af0f96f 100644 --- a/go.sum +++ b/go.sum @@ -27,7 +27,6 @@ github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdII github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845/go.mod h1:c8Mh99Cw82nrsAnPgxQSZHkswVOJF7/MqZb1ZdvriLM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= @@ -93,7 +92,6 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gernest/wow v0.1.0/go.mod h1:dEPabJRi5BneI1Nev1VWo0ZlcTWibHWp43qxKms4elY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -318,13 +316,12 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.19 h1:0ymbfaLG1/utH2+BydNiF+dx1jSEmdr/nylOtkGHZZg= -github.com/miekg/dns v1.1.19/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc= github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= -github.com/minio/minio-go/v6 v6.0.27-0.20190529152532-de69c0e465ed h1:g3DRJpu22jEjs14fSeJ7Crn9vdreiRsn4RtrEsXH/6A= -github.com/minio/minio-go/v6 v6.0.27-0.20190529152532-de69c0e465ed/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y= +github.com/minio/minio-go/v6 v6.0.39 h1:9qmKCTBpQpMdGlDAbs3mbb4mmL45/lwRUvHL1VLhYUk= +github.com/minio/minio-go/v6 v6.0.39/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -476,7 +473,6 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -489,8 +485,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49N golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -527,6 +523,8 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smto golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 h1:p9xBe/w/OzkeYVKm234g55gMdD1nSIooTir5kV11kfA= +golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -622,7 +620,6 @@ gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= From 19b9b894b649de502f499e71cb2cd3eaf3e03b7a Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Fri, 18 Oct 2019 19:52:07 +0200 Subject: [PATCH 011/257] store: Start metric and status probe HTTP server as earlier as possible (#1656) * Start metric and status probe server as soon as possible Signed-off-by: Kemal Akkoyun * Update changelog Signed-off-by: Kemal Akkoyun * Schedule a separate goroutine to start server Signed-off-by: Kemal Akkoyun * Add InitSync to the rungroup Signed-off-by: Kemal Akkoyun * Fix linter pointed issues Signed-off-by: Kemal Akkoyun * Move InitSync to alreay existed run.Group Signed-off-by: Kemal Akkoyun * Remove unnecessary changes and update CHANGELOG Signed-off-by: Kemal Akkoyun * Add simple explanation for probes Signed-off-by: Kemal Akkoyun * Make requested changes Signed-off-by: Kemal Akkoyun * Update CHANGELOG.md Co-Authored-By: Martin Chodur Signed-off-by: Kemal Akkoyun --- CHANGELOG.md | 16 +++++++---- cmd/thanos/store.go | 59 ++++++++++++++++++++++------------------ docs/components/store.md | 14 ++++++++-- 3 files changed, 54 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7c45a703b..eb96112711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. +### Fixed + +- [#1656](https://github.com/thanos-io/thanos/pull/1656) Thanos Store now starts metric and status probe HTTP server earlier in its start-up sequence. `/-/healthy` endpoint now starts to respond with success earlier. `/metrics` endpoint starts serving metrics earlier as well. Make sure to point your readiness probes to the `/-/ready` endpoint rather than `/metrics`. + ## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14 ### Fixed @@ -23,12 +27,12 @@ We use *breaking* word for marking changes that are not backward compatible (rel * NOTE: `thanos_store_nodes_grpc_connections` metric is now per `external_labels` and `store_type`. It is a recommended metric for Querier storeAPIs. `thanos_store_node_info` is marked as obsolete and will be removed in next release. * NOTE2: Store Gateway is now advertising artificial: `"@thanos_compatibility_store_type=store"` label. This is to have the current Store Gateway compatible with Querier pre v0.8.0. This label can be disabled by hidden `debug.advertise-compatibility-label=false` flag on Store Gateway. - + ## [v0.8.0](https://github.com/thanos-io/thanos/releases/tag/v0.8.0) - 2019.10.10 Lot's of improvements this release! Noteworthy items: - First Katacoda tutorial! 🐱 -- Fixed Deletion order causing Compactor to produce not needed 👻 blocks with missing random files. +- Fixed Deletion order causing Compactor to produce not needed 👻 blocks with missing random files. - Store GW memory improvements (more to come!). - Querier allows multiple deduplication labels. - Both Compactor and Store Gateway can be **sharded** within the same bucket using relabelling! @@ -42,7 +46,7 @@ both Prometheus and sidecar with Thanos: https://prometheus.io/blog/2019/10/10/r - [#1619](https://github.com/thanos-io/thanos/pull/1619) Thanos sidecar allows to limit min time range for data it exposes from Prometheus. - [#1583](https://github.com/thanos-io/thanos/pull/1583) Thanos sharding: - - Add relabel config (`--selector.relabel-config-file` and `selector.relabel-config`) into Thanos Store and Compact components. + - Add relabel config (`--selector.relabel-config-file` and `selector.relabel-config`) into Thanos Store and Compact components. Selecting blocks to serve depends on the result of block labels relabeling. - For store gateway, advertise labels from "approved" blocks. - [#1540](https://github.com/thanos-io/thanos/pull/1540) Thanos Downsample added `/-/ready` and `/-/healthy` endpoints. @@ -55,8 +59,8 @@ Selecting blocks to serve depends on the result of block labels relabeling. - [#1362](https://github.com/thanos-io/thanos/pull/1362) Optional `replicaLabels` param for `/query` and `/query_range` querier endpoints. When provided overwrite the `query.replica-label` cli flags. - [#1482](https://github.com/thanos-io/thanos/pull/1482) Thanos now supports Elastic APM as tracing provider. -- [#1612](https://github.com/thanos-io/thanos/pull/1612) Thanos Rule added `resendDelay` flag. -- [#1480](https://github.com/thanos-io/thanos/pull/1480) Thanos Receive flushes storage on hashring change. +- [#1612](https://github.com/thanos-io/thanos/pull/1612) Thanos Rule added `resendDelay` flag. +- [#1480](https://github.com/thanos-io/thanos/pull/1480) Thanos Receive flushes storage on hashring change. - [#1613](https://github.com/thanos-io/thanos/pull/1613) Thanos Receive now traces forwarded requests. ### Changed @@ -76,7 +80,7 @@ once for multiple deduplication labels like: `--query.replica-label=prometheus_r - [#1544](https://github.com/thanos-io/thanos/pull/1544) Iterating over object store is resilient to the edge case for some providers. - [#1469](https://github.com/thanos-io/thanos/pull/1469) Fixed Azure potential failures (EOF) when requesting more data then blob has. - [#1512](https://github.com/thanos-io/thanos/pull/1512) Thanos Store fixed memory leak for chunk pool. -- [#1488](https://github.com/thanos-io/thanos/pull/1488) Thanos Rule now now correctly links to query URL from rules and alerts. +- [#1488](https://github.com/thanos-io/thanos/pull/1488) Thanos Rule now now correctly links to query URL from rules and alerts. ## [v0.7.0](https://github.com/thanos-io/thanos/releases/tag/v0.7.0) - 2019.09.02 diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 139a7c8a74..70b213937e 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -126,7 +126,11 @@ func runStore( selectorRelabelConf *extflag.PathOrContent, advertiseCompatibilityLabel bool, ) error { + // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, component); err != nil { + return errors.Wrap(err, "schedule HTTP server") + } confContentYaml, err := objStoreConfig.Content() if err != nil { @@ -185,29 +189,35 @@ func runStore( return errors.Wrap(err, "create object storage store") } - begin := time.Now() - level.Debug(logger).Log("msg", "initializing bucket store") - if err := bs.InitialSync(context.Background()); err != nil { - return errors.Wrap(err, "bucket store initial sync") - } - level.Debug(logger).Log("msg", "bucket store ready", "init_duration", time.Since(begin).String()) - - ctx, cancel := context.WithCancel(context.Background()) - g.Add(func() error { - defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") - - err := runutil.Repeat(syncInterval, ctx.Done(), func() error { - if err := bs.SyncBlocks(ctx); err != nil { - level.Warn(logger).Log("msg", "syncing blocks failed", "err", err) + // bucketStoreReady signals when bucket store is ready. + bucketStoreReady := make(chan struct{}) + { + ctx, cancel := context.WithCancel(context.Background()) + g.Add(func() error { + defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") + + level.Info(logger).Log("msg", "initializing bucket store") + begin := time.Now() + if err := bs.InitialSync(ctx); err != nil { + close(bucketStoreReady) + return errors.Wrap(err, "bucket store initial sync") } - return nil + level.Info(logger).Log("msg", "bucket store ready", "init_duration", time.Since(begin).String()) + close(bucketStoreReady) + + err := runutil.Repeat(syncInterval, ctx.Done(), func() error { + if err := bs.SyncBlocks(ctx); err != nil { + level.Warn(logger).Log("msg", "syncing blocks failed", "err", err) + } + return nil + }) + + runutil.CloseWithLogOnErr(logger, bs, "bucket store") + return err + }, func(error) { + cancel() }) - - runutil.CloseWithLogOnErr(logger, bs, "bucket store") - return err - }, func(error) { - cancel() - }) + } l, err := net.Listen("tcp", grpcBindAddr) if err != nil { @@ -221,17 +231,14 @@ func runStore( s := newStoreGRPCServer(logger, reg, tracer, bs, opts) g.Add(func() error { - level.Info(logger).Log("msg", "Listening for StoreAPI gRPC", "address", grpcBindAddr) + <-bucketStoreReady + level.Info(logger).Log("msg", "listening for StoreAPI gRPC", "address", grpcBindAddr) statusProber.SetReady() return errors.Wrap(s.Serve(l), "serve gRPC") }, func(error) { s.Stop() }) - if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, component); err != nil { - return errors.Wrap(err, "schedule HTTP server") - } - level.Info(logger).Log("msg", "starting store node") return nil } diff --git a/docs/components/store.md b/docs/components/store.md index 5149fd17b4..9e5b7fe943 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -122,11 +122,11 @@ Flags: ``` -## Time based partioning +## Time based partitioning By default Thanos Store Gateway looks at all the data in Object Store and returns it based on query's time range. -Thanos Store `--min-time`, `--max-time` flags allows you to shard Thanos Store based on constant time or time duration relative to current time. +Thanos Store `--min-time`, `--max-time` flags allows you to shard Thanos Store based on constant time or time duration relative to current time. For example setting: `--min-time=-6w` & `--max-time==-2w` will make Thanos Store Gateway return metrics that fall within `now - 6 weeks` up to `now - 2 weeks` time range. @@ -136,6 +136,14 @@ Thanos Store Gateway might not get new blocks immediately, as Time partitioning We recommend having overlapping time ranges with Thanos Sidecar and other Thanos Store gateways as this will improve your resiliency to failures. -Thanos Querier deals with overlapping time series by merging them together. +Thanos Querier deals with overlapping time series by merging them together. Filtering is done on a Chunk level, so Thanos Store might still return Samples which are outside of `--min-time` & `--max-time`. + +## Probes + +- Thanos Store exposes two endpoints for probing. + - `/-/healthy` starts as soon as initial setup completed. + - `/-/ready` starts after all the bootstrapping completed (e.g initial index building) and ready to serve traffic. + +> NOTE: Metric endpoint starts immediately so, make sure you set up readiness probe on designated HTTP `/-/ready` path. From 2f88fcadba006c42a9bc35b61b9c851f18723862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Mon, 21 Oct 2019 17:52:17 +0200 Subject: [PATCH 012/257] cmd/thanos/main.go: simplify pprof registration (#1671) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit simplifies the registration of pprof HTTP endpoints. The pprof.Index handler automatically takes care of delegating to the correct handler for each profile depending on the request path: https://golang.org/src/net/http/pprof/pprof.go?s=8862:9042#L260 The following profiles are handled: https://golang.org/src/net/http/pprof/pprof.go?s=7565:8570#L248 Note that this also includes the `allocs` profile, which was previously not explicitly added. Signed-off-by: Lucas Servén Marín --- cmd/thanos/main.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/thanos/main.go b/cmd/thanos/main.go index 74550c558b..f7d39b5927 100644 --- a/cmd/thanos/main.go +++ b/cmd/thanos/main.go @@ -236,10 +236,6 @@ func registerProfile(mux *http.ServeMux) { mux.HandleFunc("/debug/pprof/profile", pprof.Profile) mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - mux.Handle("/debug/pprof/block", pprof.Handler("block")) - mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) - mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) - mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) } func registerMetrics(mux *http.ServeMux, g prometheus.Gatherer) { From 06bd4ee7bed5ef7b4098bc6b4a986bdef2f462a9 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 21 Oct 2019 17:06:35 +0100 Subject: [PATCH 013/257] Fixed compactor tests; Moved to full e2e compact test; Cleaned metrics. (#1666) * Fixed compactor tests; Moved to full e2e compact test; Cleaned metrics. Signed-off-by: Bartek Plotka * Removed block after each compaction group run. Fixes: https://github.com/thanos-io/thanos/issues/1499 Signed-off-by: Bartek Plotka * Moved to label hash for dir names for compactor groups. Fixes: https://github.com/thanos-io/thanos/issues/1661 Signed-off-by: Bartek Plotka * Addressed comments. Signed-off-by: Bartek Plotka * Addressed comments, rebased. Signed-off-by: Bartek Plotka --- CHANGELOG.md | 5 + cmd/thanos/compact.go | 8 +- cmd/thanos/downsample.go | 8 +- cmd/thanos/main_test.go | 128 +++++------ pkg/compact/compact.go | 72 ++++-- pkg/compact/compact_e2e_test.go | 352 +++++++++++++++++++++--------- pkg/compact/compact_test.go | 41 ++++ pkg/objstore/inmem/inmem.go | 16 ++ pkg/verifier/overlapped_blocks.go | 2 +- 9 files changed, 424 insertions(+), 208 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb96112711..f0074fd75c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1656](https://github.com/thanos-io/thanos/pull/1656) Thanos Store now starts metric and status probe HTTP server earlier in its start-up sequence. `/-/healthy` endpoint now starts to respond with success earlier. `/metrics` endpoint starts serving metrics earlier as well. Make sure to point your readiness probes to the `/-/ready` endpoint rather than `/metrics`. +### Changed + +- [#1666](https://github.com/thanos-io/thanos/pull/1666) `thanos_compact_group_compactions_total` now counts block compactions, so operations that resulted in a compacted block. The old behaviour +is now exposed by new metric: `thanos_compact_group_compaction_runs_started_total` and `thanos_compact_group_compaction_runs_completed_total` which counts compaction runs overall. + ## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14 ### Fixed diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 12f6573a0e..4bbb68cea2 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -344,15 +344,15 @@ func runCompact( } const ( - MetricIndexGenerateName = "thanos_compact_generated_index_total" - MetricIndexGenerateHelp = "Total number of generated indexes." + metricIndexGenerateName = "thanos_compact_generated_index_total" + metricIndexGenerateHelp = "Total number of generated indexes." ) // genMissingIndexCacheFiles scans over all blocks, generates missing index cache files and uploads them to object storage. func genMissingIndexCacheFiles(ctx context.Context, logger log.Logger, reg *prometheus.Registry, bkt objstore.Bucket, dir string) error { genIndex := prometheus.NewCounter(prometheus.CounterOpts{ - Name: MetricIndexGenerateName, - Help: MetricIndexGenerateHelp, + Name: metricIndexGenerateName, + Help: metricIndexGenerateHelp, }) reg.MustRegister(genIndex) diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index b6b2397127..ef5f3fbef6 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -214,10 +214,10 @@ func downsampleBucket( continue } if err := processDownsampling(ctx, logger, bkt, m, dir, downsample.ResLevel1); err != nil { - metrics.downsampleFailures.WithLabelValues(compact.GroupKey(*m)).Inc() + metrics.downsampleFailures.WithLabelValues(compact.GroupKey(m.Thanos)).Inc() return errors.Wrap(err, "downsampling to 5 min") } - metrics.downsamples.WithLabelValues(compact.GroupKey(*m)).Inc() + metrics.downsamples.WithLabelValues(compact.GroupKey(m.Thanos)).Inc() case downsample.ResLevel1: missing := false @@ -237,10 +237,10 @@ func downsampleBucket( continue } if err := processDownsampling(ctx, logger, bkt, m, dir, downsample.ResLevel2); err != nil { - metrics.downsampleFailures.WithLabelValues(compact.GroupKey(*m)) + metrics.downsampleFailures.WithLabelValues(compact.GroupKey(m.Thanos)) return errors.Wrap(err, "downsampling to 60 min") } - metrics.downsamples.WithLabelValues(compact.GroupKey(*m)) + metrics.downsamples.WithLabelValues(compact.GroupKey(m.Thanos)) } } return nil diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index 138ce52579..5203ce3544 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -5,6 +5,9 @@ import ( "io/ioutil" "os" "path" + "path/filepath" + + "github.com/thanos-io/thanos/pkg/block/metadata" "testing" "time" @@ -13,116 +16,105 @@ import ( "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/compact/downsample" - "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/testutil" ) -func TestCleanupCompactCacheFolder(t *testing.T) { - ctx, logger, dir, _, bkt, actReg := bootstrap(t) - defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - - sy, err := compact.NewSyncer(logger, actReg, bkt, 0*time.Second, 1, false, nil) - testutil.Ok(t, err) - - expReg := prometheus.NewRegistry() - syncExp := prometheus.NewCounter(prometheus.CounterOpts{ - Name: compact.MetricSyncMetaName, - Help: compact.MetricSyncMetaHelp, - }) - expReg.MustRegister(syncExp) - - testutil.GatherAndCompare(t, expReg, actReg, compact.MetricSyncMetaName) - - comp, err := tsdb.NewLeveledCompactor(ctx, nil, logger, []int64{1}, nil) - testutil.Ok(t, err) - - bComp, err := compact.NewBucketCompactor(logger, sy, comp, dir, bkt, 1) +func TestCleanupIndexCacheFolder(t *testing.T) { + logger := log.NewLogfmtLogger(os.Stderr) + dir, err := ioutil.TempDir("", "test-compact-cleanup") testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - // Even with with a single uploaded block the bucker compactor needs to - // downloads the meta file to plan the compaction groups. - testutil.Ok(t, bComp.Compact(ctx)) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() - syncExp.Inc() + bkt := inmem.NewBucket() - testutil.GatherAndCompare(t, expReg, actReg, compact.MetricSyncMetaName) + // Upload one compaction lvl = 2 block, one compaction lvl = 1. + // We generate index cache files only for lvl > 1 blocks. + { + id, err := testutil.CreateBlock( + ctx, + dir, + []labels.Labels{{{Name: "a", Value: "1"}}}, + 1, 0, downsample.DownsampleRange0+1, // Pass the minimum DownsampleRange0 check. + labels.Labels{{Name: "e1", Value: "1"}}, + downsample.ResLevel0) + testutil.Ok(t, err) - _, err = os.Stat(dir) - testutil.Assert(t, os.IsNotExist(err), "index cache dir shouldn't not exist at the end of execution") + meta, err := metadata.Read(filepath.Join(dir, id.String())) + testutil.Ok(t, err) -} + meta.Compaction.Level = 2 -func TestCleanupIndexCacheFolder(t *testing.T) { - ctx, logger, dir, _, bkt, actReg := bootstrap(t) - defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() + testutil.Ok(t, metadata.Write(logger, filepath.Join(dir, id.String()), meta)) + testutil.Ok(t, block.Upload(ctx, logger, bkt, path.Join(dir, id.String()))) + } + { + id, err := testutil.CreateBlock( + ctx, + dir, + []labels.Labels{{{Name: "a", Value: "1"}}}, + 1, 0, downsample.DownsampleRange0+1, // Pass the minimum DownsampleRange0 check. + labels.Labels{{Name: "e1", Value: "1"}}, + downsample.ResLevel0) + testutil.Ok(t, err) + testutil.Ok(t, block.Upload(ctx, logger, bkt, path.Join(dir, id.String()))) + } + reg := prometheus.NewRegistry() expReg := prometheus.NewRegistry() genIndexExp := prometheus.NewCounter(prometheus.CounterOpts{ - Name: MetricIndexGenerateName, - Help: MetricIndexGenerateHelp, + Name: metricIndexGenerateName, + Help: metricIndexGenerateHelp, }) expReg.MustRegister(genIndexExp) - testutil.GatherAndCompare(t, expReg, actReg, compact.MetricSyncMetaName) - - testutil.Ok(t, genMissingIndexCacheFiles(ctx, logger, actReg, bkt, dir)) + testutil.Ok(t, genMissingIndexCacheFiles(ctx, logger, reg, bkt, dir)) genIndexExp.Inc() - testutil.GatherAndCompare(t, expReg, actReg, compact.MetricSyncMetaName) - - _, err := os.Stat(dir) - testutil.Assert(t, os.IsNotExist(err), "index cache dir shouldn't not exist at the end of execution") -} - -func TestCleanupDownsampleCacheFolder(t *testing.T) { - ctx, logger, dir, blckID, bkt, reg := bootstrap(t) - defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - - meta, err := block.DownloadMeta(ctx, logger, bkt, blckID) - testutil.Ok(t, err) - - metrics := newDownsampleMetrics(reg) - testutil.Equals(t, 0.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.GroupKey(meta)))) - testutil.Ok(t, downsampleBucket(ctx, logger, metrics, bkt, dir)) - testutil.Equals(t, 1.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.GroupKey(meta)))) + testutil.GatherAndCompare(t, expReg, reg, metricIndexGenerateName) _, err = os.Stat(dir) testutil.Assert(t, os.IsNotExist(err), "index cache dir shouldn't not exist at the end of execution") } -func bootstrap(t *testing.T) (context.Context, log.Logger, string, ulid.ULID, objstore.Bucket, *prometheus.Registry) { +func TestCleanupDownsampleCacheFolder(t *testing.T) { logger := log.NewLogfmtLogger(os.Stderr) dir, err := ioutil.TempDir("", "test-compact-cleanup") testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() bkt := inmem.NewBucket() - var blckID ulid.ULID - - // Create and upload a single block to the bucker. - // The compaction will download the meta block of - // this block to plan the compaction groups. + var id ulid.ULID { - blckID, err = testutil.CreateBlock( + id, err = testutil.CreateBlock( ctx, dir, - []labels.Labels{ - {{Name: "a", Value: "1"}}, - }, + []labels.Labels{{{Name: "a", Value: "1"}}}, 1, 0, downsample.DownsampleRange0+1, // Pass the minimum DownsampleRange0 check. labels.Labels{{Name: "e1", Value: "1"}}, downsample.ResLevel0) testutil.Ok(t, err) - testutil.Ok(t, block.Upload(ctx, logger, bkt, path.Join(dir, blckID.String()))) + testutil.Ok(t, block.Upload(ctx, logger, bkt, path.Join(dir, id.String()))) } - return ctx, logger, dir, blckID, bkt, prometheus.NewRegistry() + meta, err := block.DownloadMeta(ctx, logger, bkt, id) + testutil.Ok(t, err) + + metrics := newDownsampleMetrics(prometheus.NewRegistry()) + testutil.Equals(t, 0.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.GroupKey(meta.Thanos)))) + testutil.Ok(t, downsampleBucket(ctx, logger, metrics, bkt, dir)) + testutil.Equals(t, 1.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.GroupKey(meta.Thanos)))) + + _, err = os.Stat(dir) + testutil.Assert(t, os.IsNotExist(err), "index cache dir shouldn't not exist at the end of execution") } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 348a0aa26b..01e6f78f89 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -64,22 +64,18 @@ type syncerMetrics struct { garbageCollectionFailures prometheus.Counter garbageCollectionDuration prometheus.Histogram compactions *prometheus.CounterVec + compactionRunsStarted *prometheus.CounterVec + compactionRunsCompleted *prometheus.CounterVec compactionFailures *prometheus.CounterVec } -const ( - MetricSyncMetaName = "thanos_compact_sync_meta_total" - MetricSyncMetaHelp = "Total number of sync meta operations." -) - func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { var m syncerMetrics m.syncMetas = prometheus.NewCounter(prometheus.CounterOpts{ - Name: MetricSyncMetaName, - Help: MetricSyncMetaHelp, + Name: "thanos_compact_sync_meta_total", + Help: "Total number of sync meta operations.", }) - m.syncMetaFailures = prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_compact_sync_meta_failures_total", Help: "Total number of failed sync meta operations.", @@ -96,7 +92,6 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { Name: "thanos_compact_garbage_collected_blocks_total", Help: "Total number of deleted blocks by compactor.", }) - m.garbageCollections = prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_compact_garbage_collection_total", Help: "Total number of garbage collection operations.", @@ -115,7 +110,15 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { m.compactions = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "thanos_compact_group_compactions_total", - Help: "Total number of group compactions attempts.", + Help: "Total number of group compaction attempts that resulted in a new block.", + }, []string{"group"}) + m.compactionRunsStarted = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_compact_group_compaction_runs_started_total", + Help: "Total number of group compaction attempts.", + }, []string{"group"}) + m.compactionRunsCompleted = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_compact_group_compaction_runs_completed_total", + Help: "Total number of group completed compaction runs. This also includes compactor group runs that resulted with no compaction.", }, []string{"group"}) m.compactionFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "thanos_compact_group_compactions_failures_total", @@ -132,6 +135,8 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { m.garbageCollectionFailures, m.garbageCollectionDuration, m.compactions, + m.compactionRunsStarted, + m.compactionRunsCompleted, m.compactionFailures, ) } @@ -342,12 +347,12 @@ func (c *Syncer) removeIfMetaMalformed(ctx context.Context, id ulid.ULID) (remov // GroupKey returns a unique identifier for the group the block belongs to. It considers // the downsampling resolution and the block's labels. -func GroupKey(meta metadata.Meta) string { - return groupKey(meta.Thanos.Downsample.Resolution, labels.FromMap(meta.Thanos.Labels)) +func GroupKey(meta metadata.Thanos) string { + return groupKey(meta.Downsample.Resolution, labels.FromMap(meta.Labels)) } func groupKey(res int64, lbls labels.Labels) string { - return fmt.Sprintf("%d@%s", res, lbls) + return fmt.Sprintf("%d@%v", res, lbls.Hash()) } // Groups returns the compaction groups for all blocks currently known to the syncer. @@ -358,22 +363,24 @@ func (c *Syncer) Groups() (res []*Group, err error) { groups := map[string]*Group{} for _, m := range c.blocks { - g, ok := groups[GroupKey(*m)] + g, ok := groups[GroupKey(m.Thanos)] if !ok { g, err = newGroup( - log.With(c.logger, "compactionGroup", GroupKey(*m)), + log.With(c.logger, "compactionGroup", GroupKey(m.Thanos)), c.bkt, labels.FromMap(m.Thanos.Labels), m.Thanos.Downsample.Resolution, c.acceptMalformedIndex, - c.metrics.compactions.WithLabelValues(GroupKey(*m)), - c.metrics.compactionFailures.WithLabelValues(GroupKey(*m)), + c.metrics.compactions.WithLabelValues(GroupKey(m.Thanos)), + c.metrics.compactionRunsStarted.WithLabelValues(GroupKey(m.Thanos)), + c.metrics.compactionRunsCompleted.WithLabelValues(GroupKey(m.Thanos)), + c.metrics.compactionFailures.WithLabelValues(GroupKey(m.Thanos)), c.metrics.garbageCollectedBlocks, ) if err != nil { return nil, errors.Wrap(err, "create compaction group") } - groups[GroupKey(*m)] = g + groups[GroupKey(m.Thanos)] = g res = append(res, g) } if err := g.Add(m); err != nil { @@ -513,6 +520,8 @@ type Group struct { blocks map[ulid.ULID]*metadata.Meta acceptMalformedIndex bool compactions prometheus.Counter + compactionRunsStarted prometheus.Counter + compactionRunsCompleted prometheus.Counter compactionFailures prometheus.Counter groupGarbageCollectedBlocks prometheus.Counter } @@ -525,6 +534,8 @@ func newGroup( resolution int64, acceptMalformedIndex bool, compactions prometheus.Counter, + compactionRunsStarted prometheus.Counter, + compactionRunsCompleted prometheus.Counter, compactionFailures prometheus.Counter, groupGarbageCollectedBlocks prometheus.Counter, ) (*Group, error) { @@ -539,6 +550,8 @@ func newGroup( blocks: map[ulid.ULID]*metadata.Meta{}, acceptMalformedIndex: acceptMalformedIndex, compactions: compactions, + compactionRunsStarted: compactionRunsStarted, + compactionRunsCompleted: compactionRunsCompleted, compactionFailures: compactionFailures, groupGarbageCollectedBlocks: groupGarbageCollectedBlocks, } @@ -592,8 +605,16 @@ func (cg *Group) Resolution() int64 { // Compact plans and runs a single compaction against the group. The compacted result // is uploaded into the bucket the blocks were retrieved from. func (cg *Group) Compact(ctx context.Context, dir string, comp tsdb.Compactor) (bool, ulid.ULID, error) { + cg.compactionRunsStarted.Inc() + subDir := filepath.Join(dir, cg.Key()) + defer func() { + if err := os.RemoveAll(subDir); err != nil { + level.Error(cg.logger).Log("msg", "failed to remove compaction group work directory", "path", subDir, "err", err) + } + }() + if err := os.RemoveAll(subDir); err != nil { return false, ulid.ULID{}, errors.Wrap(err, "clean compaction group dir") } @@ -604,9 +625,10 @@ func (cg *Group) Compact(ctx context.Context, dir string, comp tsdb.Compactor) ( shouldRerun, compID, err := cg.compact(ctx, subDir, comp) if err != nil { cg.compactionFailures.Inc() + return false, ulid.ULID{}, err } - cg.compactions.Inc() - return shouldRerun, compID, err + cg.compactionRunsCompleted.Inc() + return shouldRerun, compID, nil } // Issue347Error is a type wrapper for errors that should invoke repair process for broken block. @@ -829,8 +851,8 @@ func (cg *Group) compact(ctx context.Context, dir string, comp tsdb.Compactor) ( return false, ulid.ULID{}, errors.Wrapf(err, "read meta from %s", pdir) } - if cg.Key() != GroupKey(*meta) { - return false, ulid.ULID{}, halt(errors.Wrapf(err, "compact planned compaction for mixed groups. group: %s, planned block's group: %s", cg.Key(), GroupKey(*meta))) + if cg.Key() != GroupKey(meta.Thanos) { + return false, ulid.ULID{}, halt(errors.Wrapf(err, "compact planned compaction for mixed groups. group: %s, planned block's group: %s", cg.Key(), GroupKey(meta.Thanos))) } for _, s := range meta.Compaction.Sources { @@ -899,6 +921,7 @@ func (cg *Group) compact(ctx context.Context, dir string, comp tsdb.Compactor) ( // Even though this block was empty, there may be more work to do. return true, ulid.ULID{}, nil } + cg.compactions.Inc() level.Debug(cg.logger).Log("msg", "compacted blocks", "blocks", fmt.Sprintf("%v", plan), "duration", time.Since(begin)) @@ -1009,9 +1032,10 @@ func NewBucketCompactor( func (c *BucketCompactor) Compact(ctx context.Context) error { defer func() { if err := os.RemoveAll(c.compactDir); err != nil { - level.Error(c.logger).Log("msg", "failed to remove compaction cache directory", "path", c.compactDir, "err", err) + level.Error(c.logger).Log("msg", "failed to remove compaction work directory", "path", c.compactDir, "err", err) } }() + // Loop over bucket and compact until there's no work left. for { var ( @@ -1068,6 +1092,8 @@ func (c *BucketCompactor) Compact(ctx context.Context) error { level.Info(c.logger).Log("msg", "start of GC") + // Blocks that were compacted are garbage collected after each Compaction. + // However if compactor crashes we need to resolve those on startup. if err := c.sy.GarbageCollect(ctx); err != nil { return errors.Wrap(err, "garbage") } diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index ae363a5c34..6888777411 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -17,6 +17,8 @@ import ( "github.com/go-kit/kit/log" "github.com/oklog/ulid" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/index" @@ -164,148 +166,282 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) { groups, err := sy.Groups() testutil.Ok(t, err) - testutil.Equals(t, "0@{}", groups[0].Key()) + testutil.Equals(t, "0@17241709254077376921", groups[0].Key()) testutil.Equals(t, []ulid.ULID{metas[9].ULID, m3.ULID}, groups[0].IDs()) - testutil.Equals(t, "1000@{}", groups[1].Key()) + testutil.Equals(t, "1000@17241709254077376921", groups[1].Key()) testutil.Equals(t, []ulid.ULID{m4.ULID}, groups[1].IDs()) }) } -func TestGroup_Compact_e2e(t *testing.T) { - objtesting.ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { - prepareDir, err := ioutil.TempDir("", "test-compact-prepare") - testutil.Ok(t, err) - defer func() { testutil.Ok(t, os.RemoveAll(prepareDir)) }() - - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) - defer cancel() - - var metas []*metadata.Meta - extLset := labels.Labels{{Name: "e1", Value: "1"}} - b1, err := testutil.CreateBlock(ctx, prepareDir, []labels.Labels{ - {{Name: "a", Value: "1"}}, - {{Name: "a", Value: "2"}, {Name: "a", Value: "2"}}, - {{Name: "a", Value: "3"}}, - {{Name: "a", Value: "4"}}, - }, 100, 0, 1000, extLset, 124) - testutil.Ok(t, err) - - meta, err := metadata.Read(filepath.Join(prepareDir, b1.String())) - testutil.Ok(t, err) - metas = append(metas, meta) - - b3, err := testutil.CreateBlock(ctx, prepareDir, []labels.Labels{ - {{Name: "a", Value: "3"}}, - {{Name: "a", Value: "4"}}, - {{Name: "a", Value: "5"}}, - {{Name: "a", Value: "6"}}, - }, 100, 2001, 3000, extLset, 124) - testutil.Ok(t, err) - - // Mix order to make sure compact is able to deduct min time / max time. - meta, err = metadata.Read(filepath.Join(prepareDir, b3.String())) - testutil.Ok(t, err) - metas = append(metas, meta) - - // Currently TSDB does not produces empty blocks (see: https://github.com/prometheus/tsdb/pull/374). However before v2.7.0 it was - // so we still want to mimick this case as close as possible. - b2, err := createEmptyBlock(prepareDir, 1001, 2000, extLset, 124) - testutil.Ok(t, err) - - // blocks" count=3 mint=0 maxt=3000 ulid=01D1RQCRRJM77KQQ4GYDSC50GM sources="[01D1RQCRMNZBVHBPGRPG2M3NZQ 01D1RQCRPJMYN45T65YA1PRWB7 01D1RQCRNMTWJKTN5QQXFNKKH8]". +func MetricCount(c prometheus.Collector) int { + var ( + mCount int + mChan = make(chan prometheus.Metric) + done = make(chan struct{}) + ) - meta, err = metadata.Read(filepath.Join(prepareDir, b2.String())) - testutil.Ok(t, err) - metas = append(metas, meta) + go func() { + for range mChan { + mCount++ + } + close(done) + }() - // Due to TSDB compaction delay (not compacting fresh block), we need one more block to be pushed to trigger compaction. - freshB, err := testutil.CreateBlock(ctx, prepareDir, []labels.Labels{ - {{Name: "a", Value: "2"}}, - {{Name: "a", Value: "3"}}, - {{Name: "a", Value: "4"}}, - {{Name: "a", Value: "5"}}, - }, 100, 3001, 4000, extLset, 124) - testutil.Ok(t, err) + c.Collect(mChan) + close(mChan) + <-done - meta, err = metadata.Read(filepath.Join(prepareDir, freshB.String())) - testutil.Ok(t, err) - metas = append(metas, meta) + return mCount +} - // Upload and forget about tmp dir with all blocks. We want to ensure same state we will have on compactor. - testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(prepareDir, b1.String()))) - testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(prepareDir, b2.String()))) - testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(prepareDir, b3.String()))) - testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(prepareDir, freshB.String()))) +func TestGroup_Compact_e2e(t *testing.T) { + objtesting.ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() // Create fresh, empty directory for actual test. dir, err := ioutil.TempDir("", "test-compact") testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - metrics := newSyncerMetrics(nil) - g, err := newGroup( - nil, - bkt, - extLset, - 124, - false, - metrics.compactions.WithLabelValues(""), - metrics.compactionFailures.WithLabelValues(""), - metrics.garbageCollectedBlocks, - ) - testutil.Ok(t, err) + logger := log.NewLogfmtLogger(os.Stderr) - comp, err := tsdb.NewLeveledCompactor(ctx, nil, log.NewLogfmtLogger(os.Stderr), []int64{1000, 3000}, nil) - testutil.Ok(t, err) + reg := prometheus.NewRegistry() - shouldRerun, id, err := g.Compact(ctx, dir, comp) + sy, err := NewSyncer(logger, reg, bkt, 0*time.Second, 5, false, nil) testutil.Ok(t, err) - testutil.Assert(t, !shouldRerun, "group should be empty, but compactor did a compaction and told us to rerun") - - // Add all metas that would be gathered by syncMetas. - for _, m := range metas { - testutil.Ok(t, g.Add(m)) - } - shouldRerun, id, err = g.Compact(ctx, dir, comp) + comp, err := tsdb.NewLeveledCompactor(ctx, reg, logger, []int64{1000, 3000}, nil) testutil.Ok(t, err) - testutil.Assert(t, shouldRerun, "there should be compactible data, but the compactor reported there was not") - - resDir := filepath.Join(dir, id.String()) - testutil.Ok(t, block.Download(ctx, log.NewNopLogger(), bkt, id, resDir)) - meta, err = metadata.Read(resDir) + bComp, err := NewBucketCompactor(logger, sy, comp, dir, bkt, 2) testutil.Ok(t, err) - testutil.Equals(t, int64(0), meta.MinTime) - testutil.Equals(t, int64(3000), meta.MaxTime) - testutil.Equals(t, uint64(6), meta.Stats.NumSeries) - testutil.Equals(t, uint64(2*4*100), meta.Stats.NumSamples) // Only 2 times 4*100 because one block was empty. - testutil.Equals(t, 2, meta.Compaction.Level) - testutil.Equals(t, []ulid.ULID{b1, b3, b2}, meta.Compaction.Sources) - - // Check thanos meta. - testutil.Assert(t, extLset.Equals(labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") - testutil.Equals(t, int64(124), meta.Thanos.Downsample.Resolution) + // Compaction on empty should not fail. + testutil.Ok(t, bComp.Compact(ctx)) + testutil.Equals(t, 1.0, promtest.ToFloat64(sy.metrics.syncMetas)) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.syncMetaFailures)) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.garbageCollectedBlocks)) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.garbageCollectionFailures)) + testutil.Equals(t, 0, MetricCount(sy.metrics.compactions)) + testutil.Equals(t, 0, MetricCount(sy.metrics.compactionRunsStarted)) + testutil.Equals(t, 0, MetricCount(sy.metrics.compactionRunsCompleted)) + testutil.Equals(t, 0, MetricCount(sy.metrics.compactionFailures)) + + _, err = os.Stat(dir) + testutil.Assert(t, os.IsNotExist(err), "dir %s should be remove after compaction.", dir) + + // Test label name with slash, regression: https://github.com/thanos-io/thanos/issues/1661. + extLabels := labels.Labels{{Name: "e1", Value: "1/weird"}} + extLabels2 := labels.Labels{{Name: "e1", Value: "1"}} + metas := createAndUpload(t, bkt, []blockgenSpec{ + { + numSamples: 100, mint: 0, maxt: 1000, extLset: extLabels, res: 124, + series: []labels.Labels{ + {{Name: "a", Value: "1"}}, + {{Name: "a", Value: "2"}, {Name: "a", Value: "2"}}, + {{Name: "a", Value: "3"}}, + {{Name: "a", Value: "4"}}, + }, + }, + { + numSamples: 100, mint: 2000, maxt: 3000, extLset: extLabels, res: 124, + series: []labels.Labels{ + {{Name: "a", Value: "3"}}, + {{Name: "a", Value: "4"}}, + {{Name: "a", Value: "5"}}, + {{Name: "a", Value: "6"}}, + }, + }, + // Mix order to make sure compact is able to deduct min time / max time. + // Currently TSDB does not produces empty blocks (see: https://github.com/prometheus/tsdb/pull/374). However before v2.7.0 it was + // so we still want to mimick this case as close as possible. + { + mint: 1000, maxt: 2000, extLset: extLabels, res: 124, + // Empty block. + }, + // Due to TSDB compaction delay (not compacting fresh block), we need one more block to be pushed to trigger compaction. + { + numSamples: 100, mint: 3000, maxt: 4000, extLset: extLabels, res: 124, + series: []labels.Labels{ + {{Name: "a", Value: "7"}}, + }, + }, + // Extra block for "distraction" for different resolution and one for different labels. + { + numSamples: 100, mint: 5000, maxt: 6000, extLset: labels.Labels{{Name: "e1", Value: "2"}}, res: 124, + series: []labels.Labels{ + {{Name: "a", Value: "7"}}, + }, + }, + // Extra block for "distraction" for different resolution and one for different labels. + { + numSamples: 100, mint: 4000, maxt: 5000, extLset: extLabels, res: 0, + series: []labels.Labels{ + {{Name: "a", Value: "7"}}, + }, + }, + // Second group (extLabels2). + { + numSamples: 100, mint: 2000, maxt: 3000, extLset: extLabels2, res: 124, + series: []labels.Labels{ + {{Name: "a", Value: "3"}}, + {{Name: "a", Value: "4"}}, + {{Name: "a", Value: "6"}}, + }, + }, + { + numSamples: 100, mint: 0, maxt: 1000, extLset: extLabels2, res: 124, + series: []labels.Labels{ + {{Name: "a", Value: "1"}}, + {{Name: "a", Value: "2"}, {Name: "a", Value: "2"}}, + {{Name: "a", Value: "3"}}, + {{Name: "a", Value: "4"}}, + }, + }, + // Due to TSDB compaction delay (not compacting fresh block), we need one more block to be pushed to trigger compaction. + { + numSamples: 100, mint: 3000, maxt: 4000, extLset: extLabels2, res: 124, + series: []labels.Labels{ + {{Name: "a", Value: "7"}}, + }, + }, + }) - // Check object storage. All blocks that were included in new compacted one should be removed. - err = bkt.Iter(ctx, "", func(n string) error { + testutil.Ok(t, bComp.Compact(ctx)) + testutil.Equals(t, 3.0, promtest.ToFloat64(sy.metrics.syncMetas)) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.syncMetaFailures)) + testutil.Equals(t, 5.0, promtest.ToFloat64(sy.metrics.garbageCollectedBlocks)) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.garbageCollectionFailures)) + testutil.Equals(t, 4, MetricCount(sy.metrics.compactions)) + testutil.Equals(t, 1.0, promtest.ToFloat64(sy.metrics.compactions.WithLabelValues(GroupKey(metas[0].Thanos)))) + testutil.Equals(t, 1.0, promtest.ToFloat64(sy.metrics.compactions.WithLabelValues(GroupKey(metas[7].Thanos)))) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.compactions.WithLabelValues(GroupKey(metas[4].Thanos)))) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.compactions.WithLabelValues(GroupKey(metas[5].Thanos)))) + testutil.Equals(t, 4, MetricCount(sy.metrics.compactionRunsStarted)) + testutil.Equals(t, 2.0, promtest.ToFloat64(sy.metrics.compactionRunsStarted.WithLabelValues(GroupKey(metas[0].Thanos)))) + testutil.Equals(t, 2.0, promtest.ToFloat64(sy.metrics.compactionRunsStarted.WithLabelValues(GroupKey(metas[7].Thanos)))) + // TODO(bwplotka): Looks like we do some unnecessary loops. Not a major problem but investigate. + testutil.Equals(t, 2.0, promtest.ToFloat64(sy.metrics.compactionRunsStarted.WithLabelValues(GroupKey(metas[4].Thanos)))) + testutil.Equals(t, 2.0, promtest.ToFloat64(sy.metrics.compactionRunsStarted.WithLabelValues(GroupKey(metas[5].Thanos)))) + testutil.Equals(t, 4, MetricCount(sy.metrics.compactionRunsCompleted)) + testutil.Equals(t, 2.0, promtest.ToFloat64(sy.metrics.compactionRunsCompleted.WithLabelValues(GroupKey(metas[0].Thanos)))) + testutil.Equals(t, 2.0, promtest.ToFloat64(sy.metrics.compactionRunsCompleted.WithLabelValues(GroupKey(metas[7].Thanos)))) + // TODO(bwplotka): Looks like we do some unnecessary loops. Not a major problem but investigate. + testutil.Equals(t, 2.0, promtest.ToFloat64(sy.metrics.compactionRunsCompleted.WithLabelValues(GroupKey(metas[4].Thanos)))) + testutil.Equals(t, 2.0, promtest.ToFloat64(sy.metrics.compactionRunsCompleted.WithLabelValues(GroupKey(metas[5].Thanos)))) + testutil.Equals(t, 4, MetricCount(sy.metrics.compactionFailures)) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.compactionFailures.WithLabelValues(GroupKey(metas[0].Thanos)))) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.compactionFailures.WithLabelValues(GroupKey(metas[7].Thanos)))) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.compactionFailures.WithLabelValues(GroupKey(metas[4].Thanos)))) + testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.compactionFailures.WithLabelValues(GroupKey(metas[5].Thanos)))) + + _, err = os.Stat(dir) + testutil.Assert(t, os.IsNotExist(err), "dir %s should be remove after compaction.", dir) + + // Check object storage. All blocks that were included in new compacted one should be removed. New compacted ones + // are present and looks as expected. + nonCompactedExpected := map[ulid.ULID]bool{ + metas[3].ULID: false, + metas[4].ULID: false, + metas[5].ULID: false, + metas[8].ULID: false, + } + others := map[string]metadata.Meta{} + testutil.Ok(t, bkt.Iter(ctx, "", func(n string) error { id, ok := block.IsBlockDir(n) if !ok { return nil } - for _, source := range meta.Compaction.Sources { - if id.Compare(source) == 0 { - return errors.Errorf("Unexpectedly found %s block in bucket", source.String()) - } + if _, ok := nonCompactedExpected[id]; ok { + nonCompactedExpected[id] = true + return nil + } + + meta, err := block.DownloadMeta(ctx, logger, bkt, id) + if err != nil { + return err } + + others[GroupKey(meta.Thanos)] = meta return nil - }) - testutil.Ok(t, err) + })) + + for id, found := range nonCompactedExpected { + testutil.Assert(t, found, "not found expected block %s", id.String()) + } + + // We expect two compacted blocks only outside of what we expected in `nonCompactedExpected`. + testutil.Equals(t, 2, len(others)) + { + meta, ok := others[groupKey(124, extLabels)] + testutil.Assert(t, ok, "meta not found") + + testutil.Equals(t, int64(0), meta.MinTime) + testutil.Equals(t, int64(3000), meta.MaxTime) + testutil.Equals(t, uint64(6), meta.Stats.NumSeries) + testutil.Equals(t, uint64(2*4*100), meta.Stats.NumSamples) // Only 2 times 4*100 because one block was empty. + testutil.Equals(t, 2, meta.Compaction.Level) + testutil.Equals(t, []ulid.ULID{metas[0].ULID, metas[1].ULID, metas[2].ULID}, meta.Compaction.Sources) + + // Check thanos meta. + testutil.Assert(t, extLabels.Equals(labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") + testutil.Equals(t, int64(124), meta.Thanos.Downsample.Resolution) + } + { + meta, ok := others[groupKey(124, extLabels2)] + testutil.Assert(t, ok, "meta not found") + + testutil.Equals(t, int64(0), meta.MinTime) + testutil.Equals(t, int64(3000), meta.MaxTime) + testutil.Equals(t, uint64(5), meta.Stats.NumSeries) + testutil.Equals(t, uint64(2*4*100-100), meta.Stats.NumSamples) + testutil.Equals(t, 2, meta.Compaction.Level) + testutil.Equals(t, []ulid.ULID{metas[6].ULID, metas[7].ULID}, meta.Compaction.Sources) + + // Check thanos meta. + testutil.Assert(t, extLabels2.Equals(labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") + testutil.Equals(t, int64(124), meta.Thanos.Downsample.Resolution) + } }) } +type blockgenSpec struct { + mint, maxt int64 + series []labels.Labels + numSamples int + extLset labels.Labels + res int64 +} + +func createAndUpload(t testing.TB, bkt objstore.Bucket, blocks []blockgenSpec) (metas []*metadata.Meta) { + prepareDir, err := ioutil.TempDir("", "test-compact-prepare") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(prepareDir)) }() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + for _, b := range blocks { + var id ulid.ULID + var err error + if b.numSamples == 0 { + id, err = createEmptyBlock(prepareDir, b.mint, b.maxt, b.extLset, b.res) + } else { + id, err = testutil.CreateBlock(ctx, prepareDir, b.series, b.numSamples, b.mint, b.maxt, b.extLset, b.res) + } + testutil.Ok(t, err) + + meta, err := metadata.Read(filepath.Join(prepareDir, id.String())) + testutil.Ok(t, err) + metas = append(metas, meta) + + testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(prepareDir, id.String()))) + } + return metas +} + // createEmptyBlock produces empty block like it was the case before fix: https://github.com/prometheus/tsdb/pull/374. // (Prometheus pre v2.7.0). func createEmptyBlock(dir string, mint int64, maxt int64, extLset labels.Labels, resolution int64) (ulid.ULID, error) { diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index dd3f0c6806..85d0f332fd 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/relabel" @@ -105,3 +107,42 @@ func TestSyncer_SyncMetas_HandlesMalformedBlocks(t *testing.T) { testutil.Ok(t, err) testutil.Equals(t, true, exists) } + +func TestGroupKey(t *testing.T) { + for _, tcase := range []struct { + input metadata.Thanos + expected string + }{ + { + input: metadata.Thanos{}, + expected: "0@17241709254077376921", + }, + { + input: metadata.Thanos{ + Labels: map[string]string{}, + Downsample: metadata.ThanosDownsample{Resolution: 0}, + }, + expected: "0@17241709254077376921", + }, + { + input: metadata.Thanos{ + Labels: map[string]string{"foo": "bar", "foo1": "bar2"}, + Downsample: metadata.ThanosDownsample{Resolution: 0}, + }, + expected: "0@2124638872457683483", + }, + { + input: metadata.Thanos{ + Labels: map[string]string{`foo/some..thing/some.thing/../`: `a_b_c/bar-something-a\metric/a\x`}, + Downsample: metadata.ThanosDownsample{Resolution: 0}, + }, + expected: "0@16590761456214576373", + }, + } { + if ok := t.Run("", func(t *testing.T) { + testutil.Equals(t, tcase.expected, GroupKey(tcase.input)) + }); !ok { + return + } + } +} diff --git a/pkg/objstore/inmem/inmem.go b/pkg/objstore/inmem/inmem.go index 589e520dcd..edddd97e08 100644 --- a/pkg/objstore/inmem/inmem.go +++ b/pkg/objstore/inmem/inmem.go @@ -4,6 +4,7 @@ import ( "context" "io" "sort" + "sync" "bytes" "io/ioutil" @@ -16,7 +17,9 @@ import ( var errNotFound = errors.New("inmem: object not found") // Bucket implements the store.Bucket and shipper.Bucket interfaces against local memory. +// methods from Bucket interface are thread-safe. Object are assumed to be immutable. type Bucket struct { + mtx sync.RWMutex objects map[string][]byte } @@ -45,6 +48,8 @@ func (b *Bucket) Iter(_ context.Context, dir string, f func(string) error) error } dirPartsCount++ } + + b.mtx.RLock() for filename := range b.objects { if !strings.HasPrefix(filename, dir) || dir == filename { continue @@ -53,6 +58,7 @@ func (b *Bucket) Iter(_ context.Context, dir string, f func(string) error) error parts := strings.SplitAfter(filename, objstore.DirDelim) unique[strings.Join(parts[:dirPartsCount+1], "")] = struct{}{} } + b.mtx.RUnlock() var keys []string for n := range unique { @@ -86,7 +92,9 @@ func (b *Bucket) Get(_ context.Context, name string) (io.ReadCloser, error) { return nil, errors.New("inmem: object name is empty") } + b.mtx.RLock() file, ok := b.objects[name] + b.mtx.RUnlock() if !ok { return nil, errNotFound } @@ -100,7 +108,9 @@ func (b *Bucket) GetRange(_ context.Context, name string, off, length int64) (io return nil, errors.New("inmem: object name is empty") } + b.mtx.RLock() file, ok := b.objects[name] + b.mtx.RUnlock() if !ok { return nil, errNotFound } @@ -119,12 +129,16 @@ func (b *Bucket) GetRange(_ context.Context, name string, off, length int64) (io // Exists checks if the given directory exists in memory. func (b *Bucket) Exists(_ context.Context, name string) (bool, error) { + b.mtx.RLock() + defer b.mtx.RUnlock() _, ok := b.objects[name] return ok, nil } // Upload writes the file specified in src to into the memory. func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { + b.mtx.Lock() + defer b.mtx.Unlock() body, err := ioutil.ReadAll(r) if err != nil { return err @@ -135,6 +149,8 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { // Delete removes all data prefixed with the dir. func (b *Bucket) Delete(_ context.Context, name string) error { + b.mtx.Lock() + defer b.mtx.Unlock() if _, ok := b.objects[name]; !ok { return errNotFound } diff --git a/pkg/verifier/overlapped_blocks.go b/pkg/verifier/overlapped_blocks.go index 808c6f7b10..0e7be08e64 100644 --- a/pkg/verifier/overlapped_blocks.go +++ b/pkg/verifier/overlapped_blocks.go @@ -58,7 +58,7 @@ func fetchOverlaps(ctx context.Context, logger log.Logger, bkt objstore.Bucket) return err } - metas[compact.GroupKey(m)] = append(metas[compact.GroupKey(m)], m.BlockMeta) + metas[compact.GroupKey(m.Thanos)] = append(metas[compact.GroupKey(m.Thanos)], m.BlockMeta) return nil }) if err != nil { From 5bfe54eae7952478e3759a84a51dda232b0c5512 Mon Sep 17 00:00:00 2001 From: Povilas Versockas Date: Tue, 22 Oct 2019 13:36:44 +0300 Subject: [PATCH 014/257] Store: Refactor run group to errgroup (#1468) Signed-off-by: Povilas Versockas --- pkg/store/bucket.go | 53 ++++++++++++++------------------------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index d5cdf655de..2ec4f8e7ab 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -18,7 +18,6 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "github.com/oklog/run" "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -851,11 +850,12 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie req.MaxTime = s.limitMaxTime(req.MaxTime) var ( - stats = &queryStats{} - g run.Group - res []storepb.SeriesSet - mtx sync.Mutex + stats = &queryStats{} + res []storepb.SeriesSet + mtx sync.Mutex + g, ctx = errgroup.WithContext(srv.Context()) ) + s.mtx.RLock() for _, bs := range s.blockSets { @@ -873,7 +873,6 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie stats.blocksQueried++ b := b - ctx, cancel := context.WithCancel(srv.Context()) // We must keep the readers open until all their data has been sent. indexr := b.indexReader(ctx) @@ -883,7 +882,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie defer runutil.CloseWithLogOnErr(s.logger, indexr, "series block") defer runutil.CloseWithLogOnErr(s.logger, chunkr, "series block") - g.Add(func() error { + g.Go(func() error { part, pstats, err := blockSeries(ctx, b.meta.ULID, b.meta.Thanos.Labels, @@ -903,10 +902,6 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie mtx.Unlock() return nil - }, func(err error) { - if err != nil { - cancel() - } }) } } @@ -936,7 +931,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie { span, _ := tracing.StartSpan(srv.Context(), "bucket_store_preload_all") begin := time.Now() - err := g.Run() + err := g.Wait() span.Finish() if err != nil { @@ -1583,9 +1578,8 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { return uint64(ptrs[i].ptr.Start), uint64(ptrs[i].ptr.End) }) - var g run.Group + g, ctx := errgroup.WithContext(r.ctx) for _, part := range parts { - ctx, cancel := context.WithCancel(r.ctx) i, j := part.elemRng[0], part.elemRng[1] start := int64(part.start) @@ -1593,7 +1587,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { length := int64(part.end) - start // Fetch from object storage concurrently and update stats and posting list. - g.Add(func() error { + g.Go(func() error { begin := time.Now() b, err := r.block.readIndexRange(ctx, start, length) @@ -1627,14 +1621,10 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { r.stats.postingsTouchedSizeSum += len(c) } return nil - }, func(err error) { - if err != nil { - cancel() - } }) } - return g.Run() + return g.Wait() } func (r *bucketIndexReader) PreloadSeries(ids []uint64) error { @@ -1654,22 +1644,16 @@ func (r *bucketIndexReader) PreloadSeries(ids []uint64) error { parts := r.block.partitioner.Partition(len(ids), func(i int) (start, end uint64) { return ids[i], ids[i] + maxSeriesSize }) - var g run.Group - + g, ctx := errgroup.WithContext(r.ctx) for _, p := range parts { - ctx, cancel := context.WithCancel(r.ctx) s, e := p.start, p.end i, j := p.elemRng[0], p.elemRng[1] - g.Add(func() error { + g.Go(func() error { return r.loadSeries(ctx, ids[i:j], s, e) - }, func(err error) { - if err != nil { - cancel() - } }) } - return g.Run() + return g.Wait() } func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, start, end uint64) error { @@ -1829,7 +1813,7 @@ func (r *bucketChunkReader) addPreload(id uint64) error { // preload all added chunk IDs. Must be called before the first call to Chunk is made. func (r *bucketChunkReader) preload(samplesLimiter *Limiter) error { - var g run.Group + g, ctx := errgroup.WithContext(r.ctx) numChunks := uint64(0) for _, offsets := range r.preloads { @@ -1853,20 +1837,15 @@ func (r *bucketChunkReader) preload(samplesLimiter *Limiter) error { offsets := offsets for _, p := range parts { - ctx, cancel := context.WithCancel(r.ctx) s, e := uint32(p.start), uint32(p.end) m, n := p.elemRng[0], p.elemRng[1] - g.Add(func() error { + g.Go(func() error { return r.loadChunks(ctx, offsets[m:n], seq, s, e) - }, func(err error) { - if err != nil { - cancel() - } }) } } - return g.Run() + return g.Wait() } func (r *bucketChunkReader) loadChunks(ctx context.Context, offs []uint32, seq int, start, end uint32) error { From 1dc729949dda2d1fa5acc4d658f9c44f07ab06cf Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 22 Oct 2019 15:12:10 +0100 Subject: [PATCH 015/257] querier: Actually use select mint,maxt params during select. (#1675) This will definitely helps with offset queries. Same was done on https://github.com/cortexproject/cortex/pull/1012 Signed-off-by: Bartek Plotka --- pkg/query/api/v1.go | 5 +++-- pkg/query/querier.go | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/query/api/v1.go b/pkg/query/api/v1.go index 7f78587811..0152d18126 100644 --- a/pkg/query/api/v1.go +++ b/pkg/query/api/v1.go @@ -503,7 +503,8 @@ func (api *API) series(r *http.Request) (interface{}, []error, *ApiError) { } // TODO(bwplotka): Support downsampling? - q, err := api.queryableCreate(enableDedup, replicaLabels, 0, enablePartialResponse).Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end)) + q, err := api.queryableCreate(enableDedup, replicaLabels, 0, enablePartialResponse). + Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end)) if err != nil { return nil, nil, &ApiError{errorExec, err} } @@ -515,7 +516,7 @@ func (api *API) series(r *http.Request) (interface{}, []error, *ApiError) { sets []storage.SeriesSet ) for _, mset := range matcherSets { - s, warns, err := q.Select(&storage.SelectParams{}, mset...) + s, warns, err := q.Select(nil, mset...) if err != nil { return nil, nil, &ApiError{errorExec, err} } diff --git a/pkg/query/querier.go b/pkg/query/querier.go index dac3feab4b..4263d81a36 100644 --- a/pkg/query/querier.go +++ b/pkg/query/querier.go @@ -169,12 +169,18 @@ func (q *querier) Select(params *storage.SelectParams, ms ...*labels.Matcher) (s return nil, nil, errors.Wrap(err, "convert matchers") } + if params == nil { + params = &storage.SelectParams{ + Start: q.mint, + End: q.maxt, + } + } queryAggrs, resAggr := aggrsFromFunc(params.Func) resp := &seriesServer{ctx: ctx} if err := q.proxy.Series(&storepb.SeriesRequest{ - MinTime: q.mint, - MaxTime: q.maxt, + MinTime: params.Start, + MaxTime: params.End, Matchers: sms, MaxResolutionWindow: q.maxResolutionMillis, Aggregates: queryAggrs, From cb218f3df8f992b95a7d7059c05918d4e3379d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Tue, 22 Oct 2019 16:46:02 +0200 Subject: [PATCH 016/257] cmd/thanos/main_test.go: clarify backwards error (#1676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This small PR simply fixes an error message that confused me during a review. Signed-off-by: Lucas Servén Marín --- cmd/thanos/main_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index 5203ce3544..a526f0cfb1 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -81,7 +81,7 @@ func TestCleanupIndexCacheFolder(t *testing.T) { testutil.GatherAndCompare(t, expReg, reg, metricIndexGenerateName) _, err = os.Stat(dir) - testutil.Assert(t, os.IsNotExist(err), "index cache dir shouldn't not exist at the end of execution") + testutil.Assert(t, os.IsNotExist(err), "index cache dir should not exist at the end of execution") } func TestCleanupDownsampleCacheFolder(t *testing.T) { @@ -116,5 +116,5 @@ func TestCleanupDownsampleCacheFolder(t *testing.T) { testutil.Equals(t, 1.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.GroupKey(meta.Thanos)))) _, err = os.Stat(dir) - testutil.Assert(t, os.IsNotExist(err), "index cache dir shouldn't not exist at the end of execution") + testutil.Assert(t, os.IsNotExist(err), "index cache dir should not exist at the end of execution") } From 64af185e73978986c7955d2b3e8ad49e33bb099c Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 22 Oct 2019 16:22:12 +0100 Subject: [PATCH 017/257] store: Filter blocks before loading it. Sort advertise labels; Added sharding e2e test. (#1669) * store: Filter blocks before loading it. Sort advertise labels; Added sharding e2e test. Fixes: https://github.com/thanos-io/thanos/issues/1664 Signed-off-by: Bartek Plotka * Trying to speed up tests a bit. Signed-off-by: Bartek Plotka * Fixed tests. Signed-off-by: Bartek Plotka --- CHANGELOG.md | 1 + pkg/objstore/azure/azure.go | 2 +- pkg/objstore/cos/cos.go | 19 +- pkg/objstore/gcs/gcs.go | 7 +- pkg/objstore/objstore.go | 16 +- pkg/objstore/objtesting/foreach.go | 80 +++---- pkg/objstore/s3/s3.go | 13 +- pkg/objstore/swift/swift.go | 11 +- pkg/store/bucket.go | 123 +++++----- pkg/store/bucket_e2e_test.go | 2 +- pkg/store/bucket_test.go | 373 +++++++++++++++++++---------- 11 files changed, 381 insertions(+), 266 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0074fd75c..4f74eca471 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Fixed - [#1656](https://github.com/thanos-io/thanos/pull/1656) Thanos Store now starts metric and status probe HTTP server earlier in its start-up sequence. `/-/healthy` endpoint now starts to respond with success earlier. `/metrics` endpoint starts serving metrics earlier as well. Make sure to point your readiness probes to the `/-/ready` endpoint rather than `/metrics`. +- [#1669](https://github.com/thanos-io/thanos/pull/1669) Fixed store sharding. Now it does not load excluded meta.jsons and load/fetch index-cache.json files. ### Changed diff --git a/pkg/objstore/azure/azure.go b/pkg/objstore/azure/azure.go index 8f4ee867c3..83f36a81a1 100644 --- a/pkg/objstore/azure/azure.go +++ b/pkg/objstore/azure/azure.go @@ -291,7 +291,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err conf := &Config{ StorageAccountName: os.Getenv("AZURE_STORAGE_ACCOUNT"), StorageAccountKey: os.Getenv("AZURE_STORAGE_ACCESS_KEY"), - ContainerName: "thanos-e2e-test", + ContainerName: objstore.CreateTemporaryTestBucketName(t), } bc, err := yaml.Marshal(conf) diff --git a/pkg/objstore/cos/cos.go b/pkg/objstore/cos/cos.go index d81264ef0e..19fe3ceacc 100644 --- a/pkg/objstore/cos/cos.go +++ b/pkg/objstore/cos/cos.go @@ -4,19 +4,17 @@ import ( "context" "fmt" "io" - "math/rand" "net/http" "os" "strings" "testing" - "time" "github.com/go-kit/kit/log" - cos "github.com/mozillazg/go-cos" + "github.com/mozillazg/go-cos" "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/runutil" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) // DirDelim is the delimiter used to model a directory structure in an object store bucket. @@ -311,14 +309,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { t.Log("WARNING. Reusing", c.Bucket, "COS bucket for COS tests. Manual cleanup afterwards is required") return b, func() {}, nil } - - src := rand.NewSource(time.Now().UnixNano()) - - tmpBucketName := strings.Replace(fmt.Sprintf("test_%x", src.Int63()), "_", "-", -1) - if len(tmpBucketName) >= 31 { - tmpBucketName = tmpBucketName[:31] - } - c.Bucket = tmpBucketName + c.Bucket = objstore.CreateTemporaryTestBucketName(t) bc, err := yaml.Marshal(c) if err != nil { @@ -333,12 +324,12 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { if _, err := b.client.Bucket.Put(context.Background(), nil); err != nil { return nil, nil, err } - t.Log("created temporary COS bucket for COS tests with name", tmpBucketName) + t.Log("created temporary COS bucket for COS tests with name", c.Bucket) return b, func() { objstore.EmptyBucket(t, context.Background(), b) if _, err := b.client.Bucket.Delete(context.Background()); err != nil { - t.Logf("deleting bucket %s failed: %s", tmpBucketName, err) + t.Logf("deleting bucket %s failed: %s", c.Bucket, err) } }, nil } diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index 53baf1e4f7..b3ad6ef350 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -5,11 +5,9 @@ import ( "context" "fmt" "io" - "math/rand" "runtime" "strings" "testing" - "time" "cloud.google.com/go/storage" "github.com/go-kit/kit/log" @@ -19,7 +17,7 @@ import ( "golang.org/x/oauth2/google" "google.golang.org/api/iterator" "google.golang.org/api/option" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) // DirDelim is the delimiter used to model a directory structure in an object store bucket. @@ -169,9 +167,8 @@ func (b *Bucket) Close() error { func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - src := rand.NewSource(time.Now().UnixNano()) gTestConfig := Config{ - Bucket: fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), + Bucket: objstore.CreateTemporaryTestBucketName(t), } bc, err := yaml.Marshal(gTestConfig) diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index 67a0ce2a12..f231d6ef1c 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -2,10 +2,13 @@ package objstore import ( "context" + "fmt" "io" + "math/rand" "os" "path/filepath" "strings" + "testing" "time" "github.com/go-kit/kit/log" @@ -109,7 +112,7 @@ func DownloadFile(ctx context.Context, logger log.Logger, bkt BucketReader, src, rc, err := bkt.Get(ctx, src) if err != nil { - return errors.Wrap(err, "get file") + return errors.Wrapf(err, "get file %s", src) } defer runutil.CloseWithLogOnErr(logger, rc, "download block's file reader") @@ -372,3 +375,14 @@ func (rc *timingReadCloser) Read(b []byte) (n int, err error) { } return n, err } + +func CreateTemporaryTestBucketName(t testing.TB) string { + src := rand.NewSource(time.Now().UnixNano()) + + // Bucket name need to conform: https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html. + name := strings.Replace(strings.Replace(fmt.Sprintf("test_%x_%s", src.Int63(), strings.ToLower(t.Name())), "_", "-", -1), "/", "-", -1) + if len(name) >= 63 { + name = name[:63] + } + return name +} diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go index 0b5dffcb03..a2759554a0 100644 --- a/pkg/objstore/objtesting/foreach.go +++ b/pkg/objstore/objtesting/foreach.go @@ -3,9 +3,7 @@ package objtesting import ( "os" "testing" - "time" - "github.com/fortytw2/leaktest" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/cos" @@ -21,98 +19,98 @@ import ( // that deletes it after test was run. // Use THANOS_SKIP__TESTS to skip explicitly certain tests. func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) { + t.Parallel() + // Mandatory Inmem. if ok := t.Run("inmem", func(t *testing.T) { - defer leaktest.CheckTimeout(t, 10*time.Second)() - + t.Parallel() testFn(t, inmem.NewBucket()) - }); !ok { return } // Optional GCS. if _, ok := os.LookupEnv("THANOS_SKIP_GCS_TESTS"); !ok { - bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT")) - testutil.Ok(t, err) + t.Run("gcs", func(t *testing.T) { + bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT")) + testutil.Ok(t, err) + + t.Parallel() + defer closeFn() - ok := t.Run("gcs", func(t *testing.T) { // TODO(bwplotka): Add leaktest when https://github.com/GoogleCloudPlatform/google-cloud-go/issues/1025 is resolved. testFn(t, bkt) }) - closeFn() - if !ok { - return - } + } else { t.Log("THANOS_SKIP_GCS_TESTS envvar present. Skipping test against GCS.") } // Optional S3. if _, ok := os.LookupEnv("THANOS_SKIP_S3_AWS_TESTS"); !ok { - // TODO(bwplotka): Allow taking location from envvar. - bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2") - testutil.Ok(t, err) + t.Run("aws s3", func(t *testing.T) { + // TODO(bwplotka): Allow taking location from envvar. + bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2") + testutil.Ok(t, err) + + t.Parallel() + defer closeFn() - ok := t.Run("aws s3", func(t *testing.T) { // TODO(bwplotka): Add leaktest when we fix potential leak in minio library. // We cannot use leaktest for detecting our own potential leaks, when leaktest detects leaks in minio itself. // This needs to be investigated more. testFn(t, bkt) }) - closeFn() - if !ok { - return - } + } else { t.Log("THANOS_SKIP_S3_AWS_TESTS envvar present. Skipping test against S3 AWS.") } // Optional Azure. if _, ok := os.LookupEnv("THANOS_SKIP_AZURE_TESTS"); !ok { - bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests") - testutil.Ok(t, err) + t.Run("azure", func(t *testing.T) { + bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests") + testutil.Ok(t, err) + + t.Parallel() + defer closeFn() - ok := t.Run("azure", func(t *testing.T) { testFn(t, bkt) }) - closeFn() - if !ok { - return - } + } else { t.Log("THANOS_SKIP_AZURE_TESTS envvar present. Skipping test against Azure.") } // Optional SWIFT. if _, ok := os.LookupEnv("THANOS_SKIP_SWIFT_TESTS"); !ok { - container, closeFn, err := swift.NewTestContainer(t) - testutil.Ok(t, err) + t.Run("swift", func(t *testing.T) { + container, closeFn, err := swift.NewTestContainer(t) + testutil.Ok(t, err) + + t.Parallel() + defer closeFn() - ok := t.Run("swift", func(t *testing.T) { testFn(t, container) }) - closeFn() - if !ok { - return - } + } else { t.Log("THANOS_SKIP_SWIFT_TESTS envvar present. Skipping test against swift.") } // Optional COS. if _, ok := os.LookupEnv("THANOS_SKIP_TENCENT_COS_TESTS"); !ok { - bkt, closeFn, err := cos.NewTestBucket(t) - testutil.Ok(t, err) + t.Run("Tencent cos", func(t *testing.T) { + bkt, closeFn, err := cos.NewTestBucket(t) + testutil.Ok(t, err) + + t.Parallel() + defer closeFn() - ok := t.Run("Tencent cos", func(t *testing.T) { testFn(t, bkt) }) - closeFn() - if !ok { - return - } + } else { t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.") } diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index bf33400a20..2c062463d5 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -6,7 +6,6 @@ import ( "crypto/tls" "fmt" "io" - "math/rand" "net" "net/http" "os" @@ -18,7 +17,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - minio "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6" "github.com/minio/minio-go/v6/pkg/credentials" "github.com/minio/minio-go/v6/pkg/encrypt" "github.com/pkg/errors" @@ -26,7 +25,7 @@ import ( "github.com/prometheus/common/version" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/runutil" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) // DirDelim is the delimiter used to model a directory structure in an object store bucket. @@ -403,13 +402,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke } if c.Bucket == "" { - src := rand.NewSource(time.Now().UnixNano()) - - // Bucket name need to conform: https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html. - bktToCreate = strings.Replace(fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), "_", "-", -1) - if len(bktToCreate) >= 63 { - bktToCreate = bktToCreate[:63] - } + bktToCreate = objstore.CreateTemporaryTestBucketName(t) } if err := b.client.MakeBucket(bktToCreate, location); err != nil { diff --git a/pkg/objstore/swift/swift.go b/pkg/objstore/swift/swift.go index 2c06132c3c..ca74ab0f8f 100644 --- a/pkg/objstore/swift/swift.go +++ b/pkg/objstore/swift/swift.go @@ -5,11 +5,9 @@ import ( "context" "fmt" "io" - "math/rand" "os" "strings" "testing" - "time" "github.com/go-kit/kit/log" "github.com/gophercloud/gophercloud" @@ -19,7 +17,7 @@ import ( "github.com/gophercloud/gophercloud/pagination" "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/objstore" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) // DirDelim is the delimiter used to model a directory structure in an object store bucket. @@ -291,12 +289,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { return c, func() {}, nil } - src := rand.NewSource(time.Now().UnixNano()) - - tmpContainerName := fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()) - if len(tmpContainerName) >= 63 { - tmpContainerName = tmpContainerName[:63] - } + tmpContainerName := objstore.CreateTemporaryTestBucketName(t) if err := c.createContainer(tmpContainerName); err != nil { return nil, nil, err diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 2ec4f8e7ab..e1260e0ecb 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -235,7 +235,7 @@ type BucketStore struct { filterConfig *FilterConfig relabelConfig []*relabel.Config - labelSets map[uint64]labels.Labels + advLabelSets []storepb.LabelSet enableCompatibilityLabel bool } @@ -321,14 +321,14 @@ func (s *BucketStore) Close() (err error) { // It will reuse disk space as persistent cache based on s.dir param. func (s *BucketStore) SyncBlocks(ctx context.Context) error { var wg sync.WaitGroup - blockc := make(chan ulid.ULID) + blockc := make(chan *metadata.Meta) for i := 0; i < s.blockSyncConcurrency; i++ { wg.Add(1) go func() { - for id := range blockc { - if err := s.addBlock(ctx, id); err != nil { - level.Warn(s.logger).Log("msg", "loading block failed", "id", id, "err", err) + for meta := range blockc { + if err := s.addBlock(ctx, meta); err != nil { + level.Warn(s.logger).Log("msg", "loading block failed", "id", meta.ULID, "err", err) continue } } @@ -345,14 +345,27 @@ func (s *BucketStore) SyncBlocks(ctx context.Context) error { return nil } - inRange, err := s.isBlockInMinMaxRange(ctx, id) + bdir := path.Join(s.dir, id.String()) + meta, err := loadMeta(ctx, s.logger, s.bucket, bdir, id) + if err != nil { + return errors.Wrap(err, "load meta") + } + + inRange, err := s.isBlockInMinMaxRange(ctx, meta) if err != nil { level.Warn(s.logger).Log("msg", "error parsing block range", "block", id, "err", err) - return nil + return os.RemoveAll(bdir) } if !inRange { - return nil + return os.RemoveAll(bdir) + } + + // Check for block labels by relabeling. + // If output is empty, the block will be dropped. + if processedLabels := relabel.Process(promlabels.FromMap(meta.Thanos.Labels), s.relabelConfig...); processedLabels == nil { + level.Debug(s.logger).Log("msg", "ignoring block (drop in relabeling)", "block", id) + return os.RemoveAll(bdir) } allIDs[id] = struct{}{} @@ -362,7 +375,7 @@ func (s *BucketStore) SyncBlocks(ctx context.Context) error { } select { case <-ctx.Done(): - case blockc <- id: + case blockc <- meta: } return nil }) @@ -386,11 +399,19 @@ func (s *BucketStore) SyncBlocks(ctx context.Context) error { } // Sync advertise labels. + var storeLabels []storepb.Label s.mtx.Lock() - s.labelSets = make(map[uint64]labels.Labels, len(s.blocks)) - for _, bs := range s.blocks { - s.labelSets[bs.labels.Hash()] = append(labels.Labels(nil), bs.labels...) + s.advLabelSets = s.advLabelSets[:0] + for _, bs := range s.blockSets { + storeLabels := storeLabels[:0] + for _, l := range bs.labels { + storeLabels = append(storeLabels, storepb.Label{Name: l.Name, Value: l.Value}) + } + s.advLabelSets = append(s.advLabelSets, storepb.LabelSet{Labels: storeLabels}) } + sort.Slice(s.advLabelSets, func(i, j int) bool { + return strings.Compare(s.advLabelSets[i].String(), s.advLabelSets[j].String()) < 0 + }) s.mtx.Unlock() return nil @@ -431,14 +452,7 @@ func (s *BucketStore) numBlocks() int { return len(s.blocks) } -func (s *BucketStore) isBlockInMinMaxRange(ctx context.Context, id ulid.ULID) (bool, error) { - dir := filepath.Join(s.dir, id.String()) - - err, meta := loadMeta(ctx, s.logger, s.bucket, dir, id) - if err != nil { - return false, err - } - +func (s *BucketStore) isBlockInMinMaxRange(ctx context.Context, meta *metadata.Meta) (bool, error) { // We check for blocks in configured minTime, maxTime range. switch { case meta.MaxTime <= s.filterConfig.MinTime.PrometheusTimestamp(): @@ -457,8 +471,8 @@ func (s *BucketStore) getBlock(id ulid.ULID) *bucketBlock { return s.blocks[id] } -func (s *BucketStore) addBlock(ctx context.Context, id ulid.ULID) (err error) { - dir := filepath.Join(s.dir, id.String()) +func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err error) { + dir := filepath.Join(s.dir, meta.ULID.String()) defer func() { if err != nil { @@ -470,11 +484,14 @@ func (s *BucketStore) addBlock(ctx context.Context, id ulid.ULID) (err error) { }() s.metrics.blockLoads.Inc() + lset := labels.FromMap(meta.Thanos.Labels) + h := lset.Hash() + b, err := newBucketBlock( ctx, - log.With(s.logger, "block", id), + log.With(s.logger, "block", meta.ULID), + meta, s.bucket, - id, dir, s.indexCache, s.chunkPool, @@ -486,17 +503,7 @@ func (s *BucketStore) addBlock(ctx context.Context, id ulid.ULID) (err error) { s.mtx.Lock() defer s.mtx.Unlock() - lset := labels.FromMap(b.meta.Thanos.Labels) - h := lset.Hash() - - // Check for block labels by relabeling. - // If output is empty, the block will be dropped. - if processedLabels := relabel.Process(promlabels.FromMap(lset.Map()), s.relabelConfig...); processedLabels == nil { - level.Debug(s.logger).Log("msg", "dropping block(drop in relabeling)", "block", id) - return os.RemoveAll(dir) - } - b.labels = lset - sort.Sort(b.labels) + sort.Sort(lset) set, ok := s.blockSets[h] if !ok { @@ -568,14 +575,8 @@ func (s *BucketStore) Info(context.Context, *storepb.InfoRequest) (*storepb.Info } s.mtx.RLock() - res.LabelSets = make([]storepb.LabelSet, 0, len(s.labelSets)) - for _, ls := range s.labelSets { - lset := make([]storepb.Label, 0, len(ls)) - for _, l := range ls { - lset = append(lset, storepb.Label{Name: l.Name, Value: l.Value}) - } - res.LabelSets = append(res.LabelSets, storepb.LabelSet{Labels: lset}) - } + // Should we clone? + res.LabelSets = s.advLabelSets s.mtx.RUnlock() if s.enableCompatibilityLabel && len(res.LabelSets) > 0 { @@ -1187,21 +1188,18 @@ type bucketBlock struct { lvals map[string][]string postings map[labels.Label]index.Range - id ulid.ULID chunkObjs []string pendingReaders sync.WaitGroup partitioner partitioner - - labels labels.Labels } func newBucketBlock( ctx context.Context, logger log.Logger, + meta *metadata.Meta, bkt objstore.BucketReader, - id ulid.ULID, dir string, indexCache indexCache, chunkPool *pool.BytesPool, @@ -1210,23 +1208,17 @@ func newBucketBlock( b = &bucketBlock{ logger: logger, bucket: bkt, - id: id, indexCache: indexCache, chunkPool: chunkPool, dir: dir, partitioner: p, + meta: meta, } - err, meta := loadMeta(ctx, logger, bkt, dir, id) - if err != nil { - return nil, errors.Wrap(err, "load meta") - } - b.meta = meta - if err = b.loadIndexCacheFile(ctx); err != nil { return nil, errors.Wrap(err, "load index cache") } // Get object handles for all chunk files. - err = bkt.Iter(ctx, path.Join(id.String(), block.ChunksDirname), func(n string) error { + err = bkt.Iter(ctx, path.Join(meta.ULID.String(), block.ChunksDirname), func(n string) error { b.chunkObjs = append(b.chunkObjs, n) return nil }) @@ -1237,33 +1229,36 @@ func newBucketBlock( } func (b *bucketBlock) indexFilename() string { - return path.Join(b.id.String(), block.IndexFilename) + return path.Join(b.meta.ULID.String(), block.IndexFilename) } func (b *bucketBlock) indexCacheFilename() string { - return path.Join(b.id.String(), block.IndexCacheFilename) + return path.Join(b.meta.ULID.String(), block.IndexCacheFilename) } -func loadMeta(ctx context.Context, logger log.Logger, bucket objstore.BucketReader, dir string, id ulid.ULID) (error, *metadata.Meta) { +func loadMeta(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID) (*metadata.Meta, error) { // If we haven't seen the block before or it is missing the meta.json, download it. if _, err := os.Stat(path.Join(dir, block.MetaFilename)); os.IsNotExist(err) { if err := os.MkdirAll(dir, 0777); err != nil { - return errors.Wrap(err, "create dir"), nil + return nil, errors.Wrap(err, "create dir") } src := path.Join(id.String(), block.MetaFilename) - if err := objstore.DownloadFile(ctx, logger, bucket, src, dir); err != nil { - return errors.Wrap(err, "download meta.json"), nil + if err := objstore.DownloadFile(ctx, logger, bkt, src, dir); err != nil { + if bkt.IsObjNotFoundErr(errors.Cause(err)) { + level.Debug(logger).Log("msg", "meta file wasn't found. Block not ready or being deleted.", "block", id.String()) + } + return nil, errors.Wrap(err, "download meta.json") } } else if err != nil { - return err, nil + return nil, err } meta, err := metadata.Read(dir) if err != nil { - return errors.Wrap(err, "read meta.json"), nil + return nil, errors.Wrap(err, "read meta.json") } - return nil, meta + return meta, err } func (b *bucketBlock) loadIndexCacheFile(ctx context.Context) (err error) { diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index 252f3e5b93..43ce3c50b2 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -165,7 +165,7 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m if err := runutil.Repeat(100*time.Millisecond, ctx.Done(), func() error { return store.SyncBlocks(ctx) - }); err != nil && errors.Cause(err) != context.Canceled { + }); err != nil && ctx.Err() == nil { t.Fatal(err) } }() diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index c285df4ed4..69ccb57cc0 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -2,11 +2,14 @@ package store import ( "context" + "io" "io/ioutil" "math" + "os" "path" "path/filepath" "sort" + "sync" "testing" "time" @@ -453,7 +456,7 @@ func TestBucketStore_Info(t *testing.T) { testutil.Equals(t, storepb.StoreType_STORE, resp.StoreType) testutil.Equals(t, int64(math.MaxInt64), resp.MinTime) testutil.Equals(t, int64(math.MinInt64), resp.MaxTime) - testutil.Equals(t, []storepb.LabelSet{}, resp.LabelSets) + testutil.Equals(t, []storepb.LabelSet(nil), resp.LabelSets) testutil.Equals(t, []storepb.Label(nil), resp.Labels) } @@ -486,14 +489,12 @@ func TestBucketStore_isBlockInMinMaxRange(t *testing.T) { extLset, 0) testutil.Ok(t, err) - dir1, dir2 := filepath.Join(dir, id1.String()), filepath.Join(dir, id2.String()) - meta1, err := metadata.Read(dir1) + meta1, err := metadata.Read(path.Join(dir, id1.String())) testutil.Ok(t, err) - testutil.Ok(t, metadata.Write(log.NewNopLogger(), dir1, meta1)) - - meta2, err := metadata.Read(dir2) + meta2, err := metadata.Read(path.Join(dir, id2.String())) + testutil.Ok(t, err) + meta3, err := metadata.Read(path.Join(dir, id3.String())) testutil.Ok(t, err) - testutil.Ok(t, metadata.Write(log.NewNopLogger(), dir2, meta2)) // Run actual test. hourBeforeDur := prommodel.Duration(-1 * time.Hour) @@ -507,166 +508,298 @@ func TestBucketStore_isBlockInMinMaxRange(t *testing.T) { }, emptyRelabelConfig, true) testutil.Ok(t, err) - inRange, err := bucketStore.isBlockInMinMaxRange(context.TODO(), id1) + inRange, err := bucketStore.isBlockInMinMaxRange(context.TODO(), meta1) testutil.Ok(t, err) testutil.Equals(t, true, inRange) - inRange, err = bucketStore.isBlockInMinMaxRange(context.TODO(), id2) + inRange, err = bucketStore.isBlockInMinMaxRange(context.TODO(), meta2) testutil.Ok(t, err) testutil.Equals(t, true, inRange) - inRange, err = bucketStore.isBlockInMinMaxRange(context.TODO(), id3) + inRange, err = bucketStore.isBlockInMinMaxRange(context.TODO(), meta3) testutil.Ok(t, err) testutil.Equals(t, false, inRange) } -func TestBucketStore_selectorBlocks(t *testing.T) { - ctx := context.TODO() +type recorder struct { + mtx sync.Mutex + objstore.Bucket + + touched []string +} + +func (r *recorder) Get(ctx context.Context, name string) (io.ReadCloser, error) { + r.mtx.Lock() + defer r.mtx.Unlock() + + r.touched = append(r.touched, name) + return r.Bucket.Get(ctx, name) +} + +func TestBucketStore_Sharding(t *testing.T) { + ctx := context.Background() logger := log.NewNopLogger() - dir, err := ioutil.TempDir("", "selector-blocks") + + dir, err := ioutil.TempDir("", "test-sharding-prepare") testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() + bkt := inmem.NewBucket() series := []labels.Labels{labels.FromStrings("a", "1", "b", "1")} - id1, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "A"}}, 0) + id1, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0) + testutil.Ok(t, err) + testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id1.String()))) + + id2, err := testutil.CreateBlock(ctx, dir, series, 10, 1000, 2000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0) testutil.Ok(t, err) - testutil.Ok(t, objstore.UploadFile(ctx, logger, bkt, filepath.Join(dir, id1.String(), block.MetaFilename), path.Join(id1.String(), block.MetaFilename))) - testutil.Ok(t, objstore.UploadFile(ctx, logger, bkt, filepath.Join(dir, id1.String(), block.IndexFilename), path.Join(id1.String(), block.IndexFilename))) + testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id2.String()))) - id2, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "B"}}, 0) + id3, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "b"}, {Name: "region", Value: "r1"}}, 0) testutil.Ok(t, err) - testutil.Ok(t, objstore.UploadFile(ctx, logger, bkt, filepath.Join(dir, id2.String(), block.MetaFilename), path.Join(id2.String(), block.MetaFilename))) - testutil.Ok(t, objstore.UploadFile(ctx, logger, bkt, filepath.Join(dir, id2.String(), block.IndexFilename), path.Join(id2.String(), block.IndexFilename))) + testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id3.String()))) - id3, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "A"}}, 0) + id4, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r2"}}, 0) testutil.Ok(t, err) - testutil.Ok(t, objstore.UploadFile(ctx, logger, bkt, filepath.Join(dir, id3.String(), block.MetaFilename), path.Join(id3.String(), block.MetaFilename))) - testutil.Ok(t, objstore.UploadFile(ctx, logger, bkt, filepath.Join(dir, id3.String(), block.IndexFilename), path.Join(id3.String(), block.IndexFilename))) + testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id4.String()))) + if ok := t.Run("new_runs", func(t *testing.T) { + testSharding(t, "", bkt, id1, id2, id3, id4) + }); !ok { + return + } + + dir2, err := ioutil.TempDir("", "test-sharding2") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dir2)) }() + + if ok := t.Run("reuse_disk", func(t *testing.T) { + testSharding(t, dir2, bkt, id1, id2, id3, id4) + }); !ok { + return + } + +} + +func testSharding(t *testing.T, reuseDisk string, bkt objstore.Bucket, all ...ulid.ULID) { + var cached []ulid.ULID + + logger := log.NewLogfmtLogger(os.Stderr) for _, sc := range []struct { - relabelContentYaml string - exceptedLength int - exceptedIds []ulid.ULID + name string + relabel string + expectedIDs []ulid.ULID + expectedAdvLabels []storepb.LabelSet }{ { - relabelContentYaml: ` + name: "no sharding", + expectedIDs: all, + expectedAdvLabels: []storepb.LabelSet{ + { + Labels: []storepb.Label{ + {Name: "cluster", Value: "a"}, + {Name: "region", Value: "r1"}, + }, + }, + { + Labels: []storepb.Label{ + {Name: "cluster", Value: "a"}, + {Name: "region", Value: "r2"}, + }, + }, + { + Labels: []storepb.Label{ + {Name: "cluster", Value: "b"}, + {Name: "region", Value: "r1"}, + }, + }, + { + Labels: []storepb.Label{ + {Name: CompatibilityTypeLabelName, Value: "store"}, + }, + }, + }, + }, + { + name: "drop cluster=a sources", + relabel: ` - action: drop - regex: "A" + regex: "a" source_labels: - cluster `, - exceptedLength: 1, - exceptedIds: []ulid.ULID{id2}, + expectedIDs: []ulid.ULID{all[2]}, + expectedAdvLabels: []storepb.LabelSet{ + { + Labels: []storepb.Label{ + {Name: "cluster", Value: "b"}, + {Name: "region", Value: "r1"}, + }, + }, + { + Labels: []storepb.Label{ + {Name: CompatibilityTypeLabelName, Value: "store"}, + }, + }, + }, }, { - relabelContentYaml: ` + name: "keep only cluster=a sources", + relabel: ` - action: keep - regex: "A" + regex: "a" source_labels: - cluster `, - exceptedLength: 2, - exceptedIds: []ulid.ULID{id1, id3}, + expectedIDs: []ulid.ULID{all[0], all[1], all[3]}, + expectedAdvLabels: []storepb.LabelSet{ + { + Labels: []storepb.Label{ + {Name: "cluster", Value: "a"}, + {Name: "region", Value: "r1"}, + }, + }, + { + Labels: []storepb.Label{ + {Name: "cluster", Value: "a"}, + {Name: "region", Value: "r2"}, + }, + }, + { + Labels: []storepb.Label{ + {Name: CompatibilityTypeLabelName, Value: "store"}, + }, + }, + }, + }, + { + name: "keep only cluster=a without .*2 region sources", + relabel: ` + - action: keep + regex: "a" + source_labels: + - cluster + - action: drop + regex: ".*2" + source_labels: + - region + `, + expectedIDs: []ulid.ULID{all[0], all[1]}, + expectedAdvLabels: []storepb.LabelSet{ + { + Labels: []storepb.Label{ + {Name: "cluster", Value: "a"}, + {Name: "region", Value: "r1"}, + }, + }, + { + Labels: []storepb.Label{ + {Name: CompatibilityTypeLabelName, Value: "store"}, + }, + }, + }, + }, + { + name: "drop all", + relabel: ` + - action: drop + regex: "a" + source_labels: + - cluster + - action: drop + regex: "r1" + source_labels: + - region + `, + expectedIDs: []ulid.ULID{}, + expectedAdvLabels: []storepb.LabelSet(nil), }, } { - var relabelConf []*relabel.Config - err = yaml.Unmarshal([]byte(sc.relabelContentYaml), &relabelConf) - testutil.Ok(t, err) + t.Run(sc.name, func(t *testing.T) { + dir := reuseDisk + + if dir == "" { + var err error + dir, err = ioutil.TempDir("", "test-sharding") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() + } - bucketStore, err := NewBucketStore(nil, nil, bkt, dir, noopCache{}, 0, 0, 20, false, 20, - filterConf, relabelConf, true) - testutil.Ok(t, err) + var relabelConf []*relabel.Config + testutil.Ok(t, yaml.Unmarshal([]byte(sc.relabel), &relabelConf)) - for _, id := range []ulid.ULID{id1, id2, id3} { - testutil.Ok(t, bucketStore.addBlock(ctx, id)) - } - testutil.Equals(t, sc.exceptedLength, len(bucketStore.blocks)) + rec := &recorder{Bucket: bkt} + bucketStore, err := NewBucketStore(logger, nil, rec, dir, noopCache{}, 0, 0, 99, false, 20, + filterConf, relabelConf, true) + testutil.Ok(t, err) - ids := make([]ulid.ULID, 0, len(bucketStore.blocks)) - for id := range bucketStore.blocks { - ids = append(ids, id) - } - sort.Slice(sc.exceptedIds, func(i, j int) bool { - return sc.exceptedIds[i].Compare(sc.exceptedIds[j]) > 0 - }) - sort.Slice(ids, func(i, j int) bool { - return ids[i].Compare(ids[j]) > 0 - }) - testutil.Equals(t, sc.exceptedIds, ids) - } -} + testutil.Ok(t, bucketStore.SyncBlocks(context.Background())) -func TestBucketStore_InfoWithLabels(t *testing.T) { - defer leaktest.CheckTimeout(t, 10*time.Second)() + // Check "stored" blocks. + ids := make([]ulid.ULID, 0, len(bucketStore.blocks)) + for id := range bucketStore.blocks { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { + return ids[i].Compare(ids[j]) < 0 + }) + testutil.Equals(t, sc.expectedIDs, ids) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + // Check Info endpoint. + resp, err := bucketStore.Info(context.Background(), &storepb.InfoRequest{}) + testutil.Ok(t, err) - dir, err := ioutil.TempDir("", "bucketstore-test") - testutil.Ok(t, err) + testutil.Equals(t, storepb.StoreType_STORE, resp.StoreType) + testutil.Equals(t, []storepb.Label(nil), resp.Labels) + testutil.Equals(t, sc.expectedAdvLabels, resp.LabelSets) - bkt := inmem.NewBucket() - series := []labels.Labels{labels.FromStrings("a", "1", "b", "1")} + // Make sure we don't download files we did not expect to. + // Regression test: https://github.com/thanos-io/thanos/issues/1664 - logger := log.NewNopLogger() - id1, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "A"}}, 0) - testutil.Ok(t, err) - testutil.Ok(t, objstore.UploadFile(ctx, logger, bkt, filepath.Join(dir, id1.String(), block.IndexFilename), path.Join(id1.String(), block.IndexFilename))) + // Sort records. We load blocks concurrently so operations might be not ordered. + sort.Strings(rec.touched) - id2, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "B"}}, 0) - testutil.Ok(t, err) - testutil.Ok(t, objstore.UploadFile(ctx, logger, bkt, filepath.Join(dir, id2.String(), block.IndexFilename), path.Join(id2.String(), block.IndexFilename))) + if reuseDisk != "" { + testutil.Equals(t, expectedTouchedBlockOps(all, sc.expectedIDs, cached), rec.touched) + cached = sc.expectedIDs + return + } - id3, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "B"}}, 0) - testutil.Ok(t, err) - testutil.Ok(t, objstore.UploadFile(ctx, logger, bkt, filepath.Join(dir, id3.String(), block.IndexFilename), path.Join(id3.String(), block.IndexFilename))) - - relabelContentYaml := ` - - action: drop - regex: "A" - source_labels: - - cluster - ` - var relabelConfig []*relabel.Config - err = yaml.Unmarshal([]byte(relabelContentYaml), &relabelConfig) - testutil.Ok(t, err) - bucketStore, err := NewBucketStore( - nil, - nil, - bkt, - dir, - noopCache{}, - 2e5, - 0, - 0, - false, - 20, - filterConf, - relabelConfig, - true, - ) - testutil.Ok(t, err) + testutil.Equals(t, expectedTouchedBlockOps(all, sc.expectedIDs, nil), rec.touched) + }) + } +} - err = bucketStore.SyncBlocks(ctx) - testutil.Ok(t, err) +func expectedTouchedBlockOps(all []ulid.ULID, expected []ulid.ULID, cached []ulid.ULID) []string { + var ops []string + for _, id := range all { + blockCached := false + for _, fid := range cached { + if id.Compare(fid) == 0 { + blockCached = true + break + } + } + if blockCached { + continue + } - resp, err := bucketStore.Info(ctx, &storepb.InfoRequest{}) - testutil.Ok(t, err) + found := false + for _, fid := range expected { + if id.Compare(fid) == 0 { + found = true + break + } + } - testutil.Equals(t, storepb.StoreType_STORE, resp.StoreType) - testutil.Equals(t, int64(0), resp.MinTime) - testutil.Equals(t, int64(1000), resp.MaxTime) - testutil.Equals(t, []storepb.Label(nil), resp.Labels) - testutil.Equals(t, []storepb.LabelSet{ - { - Labels: []storepb.Label{ - {Name: "cluster", Value: "B"}, - }, - }, - { - Labels: []storepb.Label{ - {Name: CompatibilityTypeLabelName, Value: "store"}, - }, - }, - }, resp.LabelSets) + ops = append(ops, path.Join(id.String(), block.MetaFilename)) + if found { + ops = append(ops, + path.Join(id.String(), block.IndexCacheFilename), + path.Join(id.String(), block.IndexFilename), + ) + } + } + sort.Strings(ops) + return ops } From 7005b6507f008e67528e8b3820073a6e163cd05c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Thu, 24 Oct 2019 00:58:51 +0200 Subject: [PATCH 018/257] cmd/thanos/receive: remote-write client+server TLS (#1668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit gives the Thanos receive component the capability to use TLS in both the remote-write client and server. This means that Thanos receive can now authenticate all requests. In order to accomplish this change, this commit abstracts the majority of the logic of `defaultGRPCServerOpts` into a reusable func for gRPC and HTTP servers and creates a similar func for TLS client configurations. Signed-off-by: Lucas Servén Marín --- cmd/thanos/main.go | 67 ++++++++++++++++++++++++++++++++++++++---- cmd/thanos/query.go | 54 +++++----------------------------- cmd/thanos/receive.go | 57 +++++++++++++++++++++++++++-------- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- pkg/receive/handler.go | 14 ++++++--- 7 files changed, 125 insertions(+), 73 deletions(-) diff --git a/cmd/thanos/main.go b/cmd/thanos/main.go index f7d39b5927..f232c9b5d6 100644 --- a/cmd/thanos/main.go +++ b/cmd/thanos/main.go @@ -242,18 +242,30 @@ func registerMetrics(mux *http.ServeMux, g prometheus.Gatherer) { mux.Handle("/metrics", promhttp.HandlerFor(g, promhttp.HandlerOpts{})) } -func defaultGRPCServerOpts(logger log.Logger, cert, key, clientCA string) ([]grpc.ServerOption, error) { +func defaultGRPCTLSServerOpts(logger log.Logger, cert, key, clientCA string) ([]grpc.ServerOption, error) { opts := []grpc.ServerOption{} + tlsCfg, err := defaultTLSServerOpts(log.With(logger, "protocol", "gRPC"), cert, key, clientCA) + if err != nil { + return opts, err + } + if tlsCfg != nil { + opts = append(opts, grpc.Creds(credentials.NewTLS(tlsCfg))) + } + return opts, nil +} +func defaultTLSServerOpts(logger log.Logger, cert, key, clientCA string) (*tls.Config, error) { if key == "" && cert == "" { if clientCA != "" { return nil, errors.New("when a client CA is used a server key and certificate must also be provided") } level.Info(logger).Log("msg", "disabled TLS, key and cert must be set to enable") - return opts, nil + return nil, nil } + level.Info(logger).Log("msg", "enabling server side TLS") + if key == "" || cert == "" { return nil, errors.New("both server key and certificate must be provided") } @@ -267,8 +279,6 @@ func defaultGRPCServerOpts(logger log.Logger, cert, key, clientCA string) ([]grp return nil, errors.Wrap(err, "server credentials") } - level.Info(logger).Log("msg", "enabled gRPC server side TLS") - tlsCfg.Certificates = []tls.Certificate{tlsCert} if clientCA != "" { @@ -284,10 +294,55 @@ func defaultGRPCServerOpts(logger log.Logger, cert, key, clientCA string) ([]grp tlsCfg.ClientCAs = certPool tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert - level.Info(logger).Log("msg", "gRPC server TLS client verification enabled") + level.Info(logger).Log("msg", "server TLS client verification enabled") } - return append(opts, grpc.Creds(credentials.NewTLS(tlsCfg))), nil + return tlsCfg, nil +} + +func defaultTLSClientOpts(logger log.Logger, cert, key, caCert, serverName string) (*tls.Config, error) { + var certPool *x509.CertPool + if caCert != "" { + caPEM, err := ioutil.ReadFile(caCert) + if err != nil { + return nil, errors.Wrap(err, "reading client CA") + } + + certPool = x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caPEM) { + return nil, errors.Wrap(err, "building client CA") + } + level.Info(logger).Log("msg", "TLS client using provided certificate pool") + } else { + var err error + certPool, err = x509.SystemCertPool() + if err != nil { + return nil, errors.Wrap(err, "reading system certificate pool") + } + level.Info(logger).Log("msg", "TLS client using system certificate pool") + } + + tlsCfg := &tls.Config{ + RootCAs: certPool, + } + + if serverName != "" { + tlsCfg.ServerName = serverName + } + + if (key != "") != (cert != "") { + return nil, errors.New("both client key and certificate must be provided") + } + + if cert != "" { + cert, err := tls.LoadX509KeyPair(cert, key) + if err != nil { + return nil, errors.Wrap(err, "client credentials") + } + tlsCfg.Certificates = []tls.Certificate{cert} + level.Info(logger).Log("msg", "TLS client authentication enabled") + } + return tlsCfg, nil } func newStoreGRPCServer(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, srv storepb.StoreServer, opts []grpc.ServerOption) *grpc.Server { diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 6821e6fd36..e4c98de02b 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -2,10 +2,7 @@ package main import ( "context" - "crypto/tls" - "crypto/x509" "fmt" - "io/ioutil" "math" "net" "net/http" @@ -164,7 +161,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { } } -func storeClientGRPCOpts(logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, secure bool, cert, key, caCert string, serverName string) ([]grpc.DialOption, error) { +func storeClientGRPCOpts(logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, secure bool, cert, key, caCert, serverName string) ([]grpc.DialOption, error) { grpcMets := grpc_prometheus.NewClientMetrics() grpcMets.EnableClientHandlingTimeHistogram( grpc_prometheus.WithHistogramBuckets([]float64{ @@ -199,50 +196,13 @@ func storeClientGRPCOpts(logger log.Logger, reg *prometheus.Registry, tracer ope return append(dialOpts, grpc.WithInsecure()), nil } - level.Info(logger).Log("msg", "Enabling client to server TLS") + level.Info(logger).Log("msg", "enabling client to server TLS") - var certPool *x509.CertPool - - if caCert != "" { - caPEM, err := ioutil.ReadFile(caCert) - if err != nil { - return nil, errors.Wrap(err, "reading client CA") - } - - certPool = x509.NewCertPool() - if !certPool.AppendCertsFromPEM(caPEM) { - return nil, errors.Wrap(err, "building client CA") - } - level.Info(logger).Log("msg", "TLS Client using provided certificate pool") - } else { - var err error - certPool, err = x509.SystemCertPool() - if err != nil { - return nil, errors.Wrap(err, "reading system certificate pool") - } - level.Info(logger).Log("msg", "TLS Client using system certificate pool") - } - - tlsCfg := &tls.Config{ - RootCAs: certPool, - } - - if serverName != "" { - tlsCfg.ServerName = serverName - } - - if cert != "" { - cert, err := tls.LoadX509KeyPair(cert, key) - if err != nil { - return nil, errors.Wrap(err, "client credentials") - } - tlsCfg.Certificates = []tls.Certificate{cert} - level.Info(logger).Log("msg", "TLS Client authentication enabled") + tlsCfg, err := defaultTLSClientOpts(logger, cert, key, caCert, serverName) + if err != nil { + return nil, err } - - creds := credentials.NewTLS(tlsCfg) - - return append(dialOpts, grpc.WithTransportCredentials(creds)), nil + return append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))), nil } // runQuery starts a server that exposes PromQL Query API. It is responsible for querying configured @@ -428,7 +388,7 @@ func runQuery( } logger := log.With(logger, "component", component.Query.String()) - opts, err := defaultGRPCServerOpts(logger, srvCert, srvKey, srvClientCA) + opts, err := defaultGRPCTLSServerOpts(logger, srvCert, srvKey, srvClientCA) if err != nil { return errors.Wrap(err, "build gRPC server") } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 9639154037..a6858d3bf3 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -35,11 +35,18 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { comp := component.Receive cmd := app.Command(comp.String(), "Accept Prometheus remote write API requests and write to local tsdb (EXPERIMENTAL, this may change drastically without notice)") - grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) httpBindAddr := regHTTPAddrFlag(cmd) + grpcBindAddr, grpcCert, grpcKey, grpcClientCA := regGRPCFlags(cmd) - remoteWriteAddress := cmd.Flag("remote-write.address", "Address to listen on for remote write requests."). + rwAddress := cmd.Flag("remote-write.address", "Address to listen on for remote write requests."). Default("0.0.0.0:19291").String() + rwServerCert := cmd.Flag("remote-write.server-tls-cert", "TLS Certificate for HTTP server, leave blank to disable TLS").Default("").String() + rwServerKey := cmd.Flag("remote-write.server-tls-key", "TLS Key for the HTTP server, leave blank to disable TLS").Default("").String() + rwServerClientCA := cmd.Flag("remote-write.server-tls-client-ca", "TLS CA to verify clients against. If no client CA is specified, there is no client verification on server side. (tls.NoClientCert)").Default("").String() + rwClientCert := cmd.Flag("remote-write.client-tls-cert", "TLS Certificates to use to identify this client to the server").Default("").String() + rwClientKey := cmd.Flag("remote-write.client-tls-key", "TLS Key for the client's certificate").Default("").String() + rwClientServerCA := cmd.Flag("remote-write.client-tls-ca", "TLS CA Certificates to use to verify servers").Default("").String() + rwClientServerName := cmd.Flag("remote-write.client-server-name", "Server name to verify the hostname on the returned gRPC certificates. See https://tools.ietf.org/html/rfc4366#section-3.1").Default("").String() dataDir := cmd.Flag("tsdb.path", "Data directory of TSDB."). Default("./data").String() @@ -87,7 +94,7 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { if hostname == "" || err != nil { return errors.New("--receive.local-endpoint is empty and host could not be determined.") } - parts := strings.Split(*remoteWriteAddress, ":") + parts := strings.Split(*rwAddress, ":") port := parts[len(parts)-1] *local = fmt.Sprintf("http://%s:%s/api/v1/receive", hostname, port) } @@ -98,11 +105,18 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { reg, tracer, *grpcBindAddr, - *cert, - *key, - *clientCA, + *grpcCert, + *grpcKey, + *grpcClientCA, *httpBindAddr, - *remoteWriteAddress, + *rwAddress, + *rwServerCert, + *rwServerKey, + *rwServerClientCA, + *rwClientCert, + *rwClientKey, + *rwClientServerCA, + *rwClientServerName, *dataDir, objStoreConfig, lset, @@ -124,11 +138,18 @@ func runReceive( reg *prometheus.Registry, tracer opentracing.Tracer, grpcBindAddr string, - cert string, - key string, - clientCA string, + grpcCert string, + grpcKey string, + grpcClientCA string, httpBindAddr string, - remoteWriteAddress string, + rwAddress string, + rwServerCert string, + rwServerKey string, + rwServerClientCA string, + rwClientCert string, + rwClientKey string, + rwClientServerCA string, + rwClientServerName string, dataDir string, objStoreConfig *extflag.PathOrContent, lset labels.Labels, @@ -153,14 +174,24 @@ func runReceive( } localStorage := &tsdb.ReadyStorage{} + rwTLSConfig, err := defaultTLSServerOpts(log.With(logger, "protocol", "HTTP"), rwServerCert, rwServerKey, rwServerClientCA) + if err != nil { + return err + } + rwTLSClientConfig, err := defaultTLSClientOpts(logger, rwClientCert, rwClientKey, rwClientServerCA, rwClientServerName) + if err != nil { + return err + } webHandler := receive.NewHandler(log.With(logger, "component", "receive-handler"), &receive.Options{ - ListenAddress: remoteWriteAddress, + ListenAddress: rwAddress, Registry: reg, Endpoint: endpoint, TenantHeader: tenantHeader, ReplicaHeader: replicaHeader, ReplicationFactor: replicationFactor, Tracer: tracer, + TLSConfig: rwTLSConfig, + TLSClientConfig: rwTLSClientConfig, }) statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) @@ -317,7 +348,7 @@ func runReceive( startGRPC := make(chan struct{}) g.Add(func() error { defer close(startGRPC) - opts, err := defaultGRPCServerOpts(logger, cert, key, clientCA) + opts, err := defaultGRPCTLSServerOpts(logger, grpcCert, grpcKey, grpcClientCA) if err != nil { return errors.Wrap(err, "setup gRPC server") } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index b5cc568ebf..5e76f5bc1f 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -506,7 +506,7 @@ func runRule( store := store.NewTSDBStore(logger, reg, db, component.Rule, lset) - opts, err := defaultGRPCServerOpts(logger, cert, key, clientCA) + opts, err := defaultGRPCTLSServerOpts(logger, cert, key, clientCA) if err != nil { return errors.Wrap(err, "setup gRPC options") } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index dfc2b4ac56..416fb67d2d 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -234,7 +234,7 @@ func runSidecar( return errors.Wrap(err, "create Prometheus store") } - opts, err := defaultGRPCServerOpts(logger, cert, key, clientCA) + opts, err := defaultGRPCTLSServerOpts(logger, cert, key, clientCA) if err != nil { return errors.Wrap(err, "setup gRPC server") } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 70b213937e..6326b38104 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -224,7 +224,7 @@ func runStore( return errors.Wrap(err, "listen API address") } - opts, err := defaultGRPCServerOpts(logger, cert, key, clientCA) + opts, err := defaultGRPCTLSServerOpts(logger, cert, key, clientCA) if err != nil { return errors.Wrap(err, "grpc server options") } diff --git a/pkg/receive/handler.go b/pkg/receive/handler.go index 23bb54f280..16a988098e 100644 --- a/pkg/receive/handler.go +++ b/pkg/receive/handler.go @@ -3,6 +3,7 @@ package receive import ( "bytes" "context" + "crypto/tls" "fmt" "io/ioutil" stdlog "log" @@ -49,6 +50,8 @@ type Options struct { ReplicaHeader string ReplicationFactor uint64 Tracer opentracing.Tracer + TLSConfig *tls.Config + TLSClientConfig *tls.Config } // Handler serves a Prometheus remote write receiving HTTP endpoint. @@ -72,9 +75,11 @@ func NewHandler(logger log.Logger, o *Options) *Handler { logger = log.NewNopLogger() } - client := &http.Client{} + transport := http.DefaultTransport.(*http.Transport) + transport.TLSClientConfig = o.TLSClientConfig + client := &http.Client{Transport: transport} if o.Tracer != nil { - client.Transport = tracing.HTTPTripperware(logger, http.DefaultTransport) + client.Transport = tracing.HTTPTripperware(logger, client.Transport) } h := &Handler{ @@ -186,8 +191,9 @@ func (h *Handler) Run() error { errlog := stdlog.New(log.NewStdlibAdapter(level.Error(h.logger)), "", 0) httpSrv := &http.Server{ - Handler: nethttp.Middleware(opentracing.GlobalTracer(), mux, operationName), - ErrorLog: errlog, + Handler: nethttp.Middleware(opentracing.GlobalTracer(), mux, operationName), + ErrorLog: errlog, + TLSConfig: h.options.TLSConfig, } return httpSrv.Serve(h.listener) From 66b3d21d872c4a2e41c9b2e1bb802d4ced069725 Mon Sep 17 00:00:00 2001 From: Jinhu Wu Date: Thu, 24 Oct 2019 18:59:27 +0800 Subject: [PATCH 019/257] objstore : implement Aliyun OSS (#1573) * add oss support Signed-off-by: wujinhu * fix docs Signed-off-by: wujinhu * fix Makefile Signed-off-by: wujinhu * review comments Signed-off-by: wujinhu * fix style Signed-off-by: wujinhu * review comments Signed-off-by: wujinhu * review comments Signed-off-by: wujinhu * review comments Signed-off-by: wujinhu * review comments Signed-off-by: wujinhu --- CHANGELOG.md | 1 + CONTRIBUTING.md | 1 + Makefile | 8 +- docs/storage.md | 18 ++ go.mod | 2 + go.sum | 4 + pkg/objstore/client/factory.go | 14 +- pkg/objstore/objtesting/foreach.go | 18 ++ pkg/objstore/oss/oss.go | 341 +++++++++++++++++++++++++++++ scripts/cfggen/main.go | 12 +- 10 files changed, 406 insertions(+), 13 deletions(-) create mode 100644 pkg/objstore/oss/oss.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f74eca471..331a702fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Added - [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. +- [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss-configuration) for further information. ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e589fb3d50..b6b6962505 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,6 +57,7 @@ $ git push origin - THANOS_SKIP_AZURE_TESTS to skip Azure tests. - THANOS_SKIP_SWIFT_TESTS to skip SWIFT tests. - THANOS_SKIP_TENCENT_COS_TESTS to skip Tencent COS tests. +- THANOS_SKIP_ALIYUN_OSS_TESTS to skip Aliyun OSS tests. If you skip all of these, the store specific tests will be run against memory object storage only. CI runs GCS and inmem tests only for now. Not having these variables will produce auth errors against GCS, AWS, Azure or COS tests. diff --git a/Makefile b/Makefile index 0f81720236..b9de15fc2b 100644 --- a/Makefile +++ b/Makefile @@ -160,8 +160,8 @@ docs: $(EMBEDMD) build .PHONY: check-docs check-docs: $(EMBEDMD) $(LICHE) build @EMBEDMD_BIN="$(EMBEDMD)" scripts/genflagdocs.sh check - @$(LICHE) --recursive docs --exclude "cloud.tencent.com" --document-root . - @$(LICHE) --exclude "cloud.tencent.com|goreportcard.com" --document-root . *.md + @$(LICHE) --recursive docs --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root . + @$(LICHE) --exclude "(cloud.tencent.com|goreportcard.com|alibabacloud.com)" --document-root . *.md # checks Go code comments if they have trailing period (excludes protobuffers and vendor files). # Comments with more than 3 spaces at beginning are omitted from the check, example: '// - foo'. @@ -203,17 +203,19 @@ test: check-git install-deps @go install github.com/thanos-io/thanos/cmd/thanos # Be careful on GOCACHE. Those tests are sometimes using built Thanos/Prometheus binaries directly. Don't cache those. @rm -rf ${GOCACHE} - @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets" + @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_ALIYUN_OSS_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets" @go test $(shell go list ./... | grep -v /vendor/); .PHONY: test-ci test-ci: export THANOS_SKIP_AZURE_TESTS = true test-ci: export THANOS_SKIP_SWIFT_TESTS = true test-ci: export THANOS_SKIP_TENCENT_COS_TESTS = true +test-ci: export THANOS_SKIP_ALIYUN_OSS_TESTS = true test-ci: @echo ">> Skipping AZURE tests" @echo ">> Skipping SWIFT tests" @echo ">> Skipping TENCENT tests" + @echo ">> Skipping ALIYUN tests" $(MAKE) test .PHONY: test-local diff --git a/docs/storage.md b/docs/storage.md index 20d815a439..2abd2b5ea1 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -60,6 +60,7 @@ Current object storage client implementations: | [Azure Storage Account](./storage.md#azure) | Stable (production usage) | yes | @vglafirov | | [OpenStack Swift](./storage.md#openstack-swift) | Beta (working PoCs, testing usage) | no | @sudhi-vm | | [Tencent COS](./storage.md#tencent-cos) | Beta (testing usage) | no | @jojohappy | +| [AliYun OSS](./storage.md#aliyun-oss) | Beta (testing usage) | no | @shaulboozhiao,@wujinhu | NOTE: Currently Thanos requires strong consistency (write-read) for object store implementation. @@ -336,3 +337,20 @@ config: ``` Set the flags `--objstore.config-file` to reference to the configuration file. + +## AliYun OSS Configuration +In order to use AliYun OSS object storage, you should first create a bucket with proper Storage Class , ACLs and get the access key on the AliYun cloud. Go to [https://www.alibabacloud.com/product/oss](https://www.alibabacloud.com/product/oss) for more detail. + +To use AliYun OSS object storage, please specify following yaml configuration file in `objstore.config*` flag. + +[embedmd]:# (flags/config_bucket_aliyunoss.txt $) +```$ +type: ALIYUNOSS +config: + endpoint: "" + bucket: "" + access_key_id: "" + access_key_secret: "" +``` + +Use --objstore.config-file to reference to this configuration file. diff --git a/go.mod b/go.mod index a9cd7f1f63..7409165622 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ require ( cloud.google.com/go v0.44.1 github.com/Azure/azure-storage-blob-go v0.7.0 github.com/NYTimes/gziphandler v1.1.1 + github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 + github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/fatih/structtag v1.0.0 diff --git a/go.sum b/go.sum index 655af0f96f..312e68f3cf 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZq github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible h1:/MzpJOMHn/uBtd1dkS7Q9PF2ZjT6xTQMXSvv1e6ydXc= +github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -47,6 +49,8 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.23.12 h1:2UnxgNO6Y5J1OrkXS8XNp0UatDxD1bWHiDT62RDPggI= github.com/aws/aws-sdk-go v1.23.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= +github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 2a4edc86b0..50caf4d8e7 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -13,6 +13,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/cos" "github.com/thanos-io/thanos/pkg/objstore/gcs" + "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" yaml "gopkg.in/yaml.v2" @@ -21,11 +22,12 @@ import ( type ObjProvider string const ( - GCS ObjProvider = "GCS" - S3 ObjProvider = "S3" - AZURE ObjProvider = "AZURE" - SWIFT ObjProvider = "SWIFT" - COS ObjProvider = "COS" + GCS ObjProvider = "GCS" + S3 ObjProvider = "S3" + AZURE ObjProvider = "AZURE" + SWIFT ObjProvider = "SWIFT" + COS ObjProvider = "COS" + ALIYUNOSS ObjProvider = "ALIYUNOSS" ) type BucketConfig struct { @@ -59,6 +61,8 @@ func NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registe bucket, err = swift.NewContainer(logger, config) case string(COS): bucket, err = cos.NewBucket(logger, config, component) + case string(ALIYUNOSS): + bucket, err = oss.NewBucket(logger, config, component) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go index a2759554a0..29f09b4258 100644 --- a/pkg/objstore/objtesting/foreach.go +++ b/pkg/objstore/objtesting/foreach.go @@ -9,6 +9,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/cos" "github.com/thanos-io/thanos/pkg/objstore/gcs" "github.com/thanos-io/thanos/pkg/objstore/inmem" + "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" "github.com/thanos-io/thanos/pkg/testutil" @@ -114,4 +115,21 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) } else { t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.") } + + // Optional OSS. + if _, ok := os.LookupEnv("THANOS_SKIP_ALIYUN_OSS_TESTS"); !ok { + bkt, closeFn, err := oss.NewTestBucket(t) + testutil.Ok(t, err) + + ok := t.Run("AliYun oss", func(t *testing.T) { + testFn(t, bkt) + }) + + closeFn() + if !ok { + return + } + } else { + t.Log("THANOS_SKIP_ALIYUN_OSS_TESTS envvar present. Skipping test against AliYun OSS.") + } } diff --git a/pkg/objstore/oss/oss.go b/pkg/objstore/oss/oss.go new file mode 100644 index 0000000000..bd913f6c96 --- /dev/null +++ b/pkg/objstore/oss/oss.go @@ -0,0 +1,341 @@ +package oss + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "math" + "math/rand" + "net/http" + "os" + "strconv" + "strings" + "testing" + "time" + + alioss "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/go-kit/kit/log" + "github.com/pkg/errors" + "github.com/thanos-io/thanos/pkg/objstore" + "gopkg.in/yaml.v2" +) + +// Part size for multi part upload. +const PartSize = 1024 * 1024 * 128 + +// Config stores the configuration for oss bucket. +type Config struct { + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` + AccessKeyID string `yaml:"access_key_id"` + AccessKeySecret string `yaml:"access_key_secret"` +} + +// Bucket implements the store.Bucket interface. +type Bucket struct { + name string + logger log.Logger + client *alioss.Client + config Config + bucket *alioss.Bucket +} + +func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { + c := Config{ + Endpoint: os.Getenv("ALIYUNOSS_ENDPOINT"), + Bucket: os.Getenv("ALIYUNOSS_BUCKET"), + AccessKeyID: os.Getenv("ALIYUNOSS_ACCESS_KEY_ID"), + AccessKeySecret: os.Getenv("ALIYUNOSS_ACCESS_KEY_SECRET"), + } + + if c.Endpoint == "" || c.AccessKeyID == "" || c.AccessKeySecret == "" { + return nil, nil, errors.New("aliyun oss endpoint or access_key_id or access_key_secret " + + "is not present in config file") + } + if c.Bucket != "" && os.Getenv("THANOS_ALLOW_EXISTING_BUCKET_USE") == "true" { + t.Log("ALIYUNOSS_BUCKET is defined. Normally this tests will create temporary bucket " + + "and delete it after test. Unset ALIYUNOSS_BUCKET env variable to use default logic. If you really want to run " + + "tests against provided (NOT USED!) bucket, set THANOS_ALLOW_EXISTING_BUCKET_USE=true.") + return NewTestBucketFromConfig(t, c, true) + } + return NewTestBucketFromConfig(t, c, false) +} + +func calculateChunks(name string, r io.Reader) (int, int64, error) { + switch r.(type) { + case *os.File: + f, _ := r.(*os.File) + if fileInfo, err := f.Stat(); err == nil { + s := fileInfo.Size() + return int(math.Floor(float64(s) / PartSize)), s % PartSize, nil + } + case *strings.Reader: + f, _ := r.(*strings.Reader) + return int(math.Floor(float64(f.Size()) / PartSize)), f.Size() % PartSize, nil + } + return -1, 0, errors.New("unsupported implement of io.Reader") +} + +// Upload the contents of the reader as an object into the bucket. +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { + chunksnum, lastslice, err := calculateChunks(name, r) + if err != nil { + return err + } + + ncloser := ioutil.NopCloser(r) + switch chunksnum { + case 0: + if err := b.bucket.PutObject(name, ncloser); err != nil { + return errors.Wrap(err, "failed to upload oss object") + } + default: + { + init, err := b.bucket.InitiateMultipartUpload(name) + if err != nil { + return errors.Wrap(err, "failed to initiate multi-part upload") + } + chunk := 0 + uploadEveryPart := func(everypartsize int64, cnk int) (alioss.UploadPart, error) { + prt, err := b.bucket.UploadPart(init, ncloser, everypartsize, cnk) + if err != nil { + if err := b.bucket.AbortMultipartUpload(init); err != nil { + return prt, errors.Wrap(err, "failed to abort multi-part upload") + } + + return prt, errors.Wrap(err, "failed to upload multi-part chunk") + } + return prt, nil + } + var parts []alioss.UploadPart + for ; chunk < chunksnum; chunk++ { + part, err := uploadEveryPart(PartSize, chunk+1) + if err != nil { + return errors.Wrap(err, "failed to upload every part") + } + parts = append(parts, part) + } + if lastslice != 0 { + part, err := uploadEveryPart(lastslice, chunksnum+1) + if err != nil { + return errors.Wrap(err, "failed to upload the last chunk") + } + parts = append(parts, part) + } + if _, err := b.bucket.CompleteMultipartUpload(init, parts); err != nil { + return errors.Wrap(err, "failed to set multi-part upload completive") + } + } + } + return nil +} + +// Delete removes the object with the given name. +func (b *Bucket) Delete(ctx context.Context, name string) error { + if err := b.bucket.DeleteObject(name); err != nil { + return errors.Wrap(err, "delete oss object") + } + return nil +} + +// NewBucket returns a new Bucket using the provided oss config values. +func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { + var config Config + if err := yaml.Unmarshal(conf, &config); err != nil { + return nil, errors.Wrap(err, "parse aliyun oss config file failed") + } + + if config.Endpoint == "" || config.Bucket == "" || config.AccessKeyID == "" || config.AccessKeySecret == "" { + return nil, errors.New("aliyun oss endpoint or bucket or access_key_id or access_key_secret " + + "is not present in config file") + } + + client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) + if err != nil { + return nil, errors.Wrap(err, "create aliyun oss client failed") + } + bk, err := client.Bucket(config.Bucket) + if err != nil { + return nil, errors.Wrapf(err, "use aliyun oss bucket %s failed", config.Bucket) + } + + bkt := &Bucket{ + logger: logger, + client: client, + name: config.Bucket, + config: config, + bucket: bk, + } + return bkt, nil +} + +// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) error { + if dir != "" { + dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim + } + + marker := alioss.Marker("") + for { + if err := ctx.Err(); err != nil { + return errors.Wrap(err, "context closed while iterating bucket") + } + objects, err := b.bucket.ListObjects(alioss.Prefix(dir), alioss.Delimiter(objstore.DirDelim), marker) + if err != nil { + return errors.Wrap(err, "listing aliyun oss bucket failed") + } + marker = alioss.Marker(objects.NextMarker) + + for _, object := range objects.Objects { + if err := f(object.Key); err != nil { + return errors.Wrapf(err, "callback func invoke for object %s failed ", object.Key) + } + } + + for _, object := range objects.CommonPrefixes { + if err := f(object); err != nil { + return errors.Wrapf(err, "callback func invoke for directory %s failed", object) + } + } + if !objects.IsTruncated { + break + } + } + + return nil +} + +func (b *Bucket) Name() string { + return b.name +} + +func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool) (objstore.Bucket, func(), error) { + if c.Bucket == "" { + src := rand.NewSource(time.Now().UnixNano()) + + bktToCreate := strings.Replace(fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), "_", "-", -1) + if len(bktToCreate) >= 63 { + bktToCreate = bktToCreate[:63] + } + testclient, err := alioss.New(c.Endpoint, c.AccessKeyID, c.AccessKeySecret) + if err != nil { + return nil, nil, errors.Wrap(err, "create aliyun oss client failed") + } + + if err := testclient.CreateBucket(bktToCreate); err != nil { + return nil, nil, errors.Wrapf(err, "create aliyun oss bucket %s failed", bktToCreate) + } + c.Bucket = bktToCreate + } + + bc, err := yaml.Marshal(c) + if err != nil { + return nil, nil, err + } + + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test") + if err != nil { + return nil, nil, err + } + + if reuseBucket { + if err := b.Iter(context.Background(), "", func(f string) error { + return errors.Errorf("bucket %s is not empty", c.Bucket) + }); err != nil { + return nil, nil, errors.Wrapf(err, "oss check bucket %s", c.Bucket) + } + + t.Log("WARNING. Reusing", c.Bucket, "Aliyun OSS bucket for OSS tests. Manual cleanup afterwards is required") + return b, func() {}, nil + } + + return b, func() { + objstore.EmptyBucket(t, context.Background(), b) + if err := b.client.DeleteBucket(c.Bucket); err != nil { + t.Logf("deleting bucket %s failed: %s", c.Bucket, err) + } + }, nil +} + +func (b *Bucket) Close() error { return nil } + +func (b *Bucket) setRange(start, end int64, name string) (alioss.Option, error) { + var opt alioss.Option + if 0 <= start && start <= end { + header, err := b.bucket.GetObjectMeta(name) + if err != nil { + return nil, err + } + + size, err := strconv.ParseInt(header["Content-Length"][0], 10, 0) + if err != nil { + return nil, err + } + + if end > size { + end = size - 1 + } + + opt = alioss.Range(start, end) + } else { + return nil, errors.Errorf("Invalid range specified: start=%d end=%d", start, end) + } + return opt, nil +} + +func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { + if len(name) == 0 { + return nil, errors.New("given object name should not empty") + } + + var opts []alioss.Option + if length != -1 { + opt, err := b.setRange(off, off+length-1, name) + if err != nil { + return nil, err + } + opts = append(opts, opt) + } + + resp, err := b.bucket.GetObject(name, opts...) + if err != nil { + return nil, err + } + + return resp, nil +} + +// Get returns a reader for the given object name. +func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { + return b.getRange(ctx, name, 0, -1) +} + +func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { + return b.getRange(ctx, name, off, length) +} + +// Exists checks if the given object exists in the bucket. +func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { + exists, err := b.bucket.IsObjectExist(name) + if err != nil { + if b.IsObjNotFoundErr(err) { + return false, nil + } + return false, errors.Wrap(err, "cloud not check if object exists") + } + + return exists, nil +} + +// IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. +func (b *Bucket) IsObjNotFoundErr(err error) bool { + switch aliErr := err.(type) { + case alioss.ServiceError: + if aliErr.StatusCode == http.StatusNotFound { + return true + } + } + return false +} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 4bf93a69c8..7b79819415 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -16,6 +16,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/cos" "github.com/thanos-io/thanos/pkg/objstore/gcs" + "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" trclient "github.com/thanos-io/thanos/pkg/tracing/client" @@ -28,11 +29,12 @@ import ( var ( bucketConfigs = map[client.ObjProvider]interface{}{ - client.AZURE: azure.Config{}, - client.GCS: gcs.Config{}, - client.S3: s3.DefaultConfig, - client.SWIFT: swift.SwiftConfig{}, - client.COS: cos.Config{}, + client.AZURE: azure.Config{}, + client.GCS: gcs.Config{}, + client.S3: s3.DefaultConfig, + client.SWIFT: swift.SwiftConfig{}, + client.COS: cos.Config{}, + client.ALIYUNOSS: oss.Config{}, } tracingConfigs = map[trclient.TracingProvider]interface{}{ trclient.JAEGER: jaeger.Config{}, From 9c2fe0392a40cce0012f64ce297dfa1c51926f62 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Thu, 24 Oct 2019 18:28:55 +0200 Subject: [PATCH 020/257] .*: Add new http-grace-period flag (#1680) * Add new http-grace-period flag Signed-off-by: Kemal Akkoyun * Update CHANGELOG Signed-off-by: Kemal Akkoyun * Update docs Signed-off-by: Kemal Akkoyun * Update pkg/server/http.go Co-Authored-By: Bartlomiej Plotka Signed-off-by: Kemal Akkoyun * Rename initializer for HTTP server Signed-off-by: Kemal Akkoyun --- CHANGELOG.md | 1 + cmd/thanos/compact.go | 13 +++-- cmd/thanos/downsample.go | 13 +++-- cmd/thanos/flags.go | 14 +++-- cmd/thanos/main.go | 46 ----------------- cmd/thanos/query.go | 14 +++-- cmd/thanos/receive.go | 12 +++-- cmd/thanos/rule.go | 14 +++-- cmd/thanos/sidecar.go | 15 ++++-- cmd/thanos/store.go | 13 +++-- docs/components/compact.md | 2 + docs/components/query.md | 2 + docs/components/rule.md | 2 + docs/components/sidecar.md | 2 + docs/components/store.md | 2 + pkg/server/http.go | 102 +++++++++++++++++++++++++++++++++++++ pkg/server/option.go | 33 ++++++++++++ 17 files changed, 226 insertions(+), 74 deletions(-) create mode 100644 pkg/server/http.go create mode 100644 pkg/server/option.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 331a702fa3..b40e93880e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. - [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss-configuration) for further information. +- [#1680](https://github.com/thanos-io/thanos/pull/1680) Add a new `--http-grace-period` CLI option to components which serve HTTP to set how long to wait until HTTP Server shuts down. ### Fixed diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 4bbb68cea2..09ea8fba93 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -11,6 +11,7 @@ import ( "time" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/server" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -79,6 +80,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) { Hidden().Default("false").Bool() httpAddr := regHTTPAddrFlag(cmd) + httpGracePeriod := regHTTPGracePeriodFlag(cmd) dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() @@ -116,6 +118,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) { m[component.Compact.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { return runCompact(g, logger, reg, *httpAddr, + time.Duration(*httpGracePeriod), *dataDir, objStoreConfig, time.Duration(*consistencyDelay), @@ -143,6 +146,7 @@ func runCompact( logger log.Logger, reg *prometheus.Registry, httpBindAddr string, + httpGracePeriod time.Duration, dataDir string, objStoreConfig *extflag.PathOrContent, consistencyDelay time.Duration, @@ -175,9 +179,12 @@ func runCompact( statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, component); err != nil { - return errors.Wrap(err, "schedule HTTP server with probes") - } + srv := server.NewHTTP(logger, reg, component, statusProber, + server.WithListen(httpBindAddr), + server.WithGracePeriod(httpGracePeriod), + ) + + g.Add(srv.ListenAndServe, srv.Shutdown) confContentYaml, err := objStoreConfig.Content() if err != nil { diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index ef5f3fbef6..9e360cec67 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -7,6 +7,7 @@ import ( "time" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/server" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -34,6 +35,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(comp.String(), "continuously downsamples blocks in an object store bucket") httpAddr := regHTTPAddrFlag(cmd) + httpGracePeriod := regHTTPGracePeriodFlag(cmd) dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() @@ -41,7 +43,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application) { objStoreConfig := regCommonObjStoreFlags(cmd, "", true) m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { - return runDownsample(g, logger, reg, *httpAddr, *dataDir, objStoreConfig, comp) + return runDownsample(g, logger, reg, *httpAddr, time.Duration(*httpGracePeriod), *dataDir, objStoreConfig, comp) } } @@ -73,6 +75,7 @@ func runDownsample( logger log.Logger, reg *prometheus.Registry, httpBindAddr string, + httpGracePeriod time.Duration, dataDir string, objStoreConfig *extflag.PathOrContent, comp component.Component, @@ -123,9 +126,11 @@ func runDownsample( } // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, comp); err != nil { - return errors.Wrap(err, "schedule HTTP server with probe") - } + srv := server.NewHTTP(logger, reg, comp, statusProber, + server.WithListen(httpBindAddr), + server.WithGracePeriod(httpGracePeriod), + ) + g.Add(srv.ListenAndServe, srv.Shutdown) level.Info(logger).Log("msg", "starting downsample node") return nil diff --git a/cmd/thanos/flags.go b/cmd/thanos/flags.go index f8cd6570cc..135552e6f6 100644 --- a/cmd/thanos/flags.go +++ b/cmd/thanos/flags.go @@ -10,6 +10,13 @@ import ( "gopkg.in/alecthomas/kingpin.v2" ) +func modelDuration(flags *kingpin.FlagClause) *model.Duration { + value := new(model.Duration) + flags.SetValue(value) + + return value +} + func regGRPCFlags(cmd *kingpin.CmdClause) ( grpcBindAddr *string, grpcTLSSrvCert *string, @@ -33,11 +40,8 @@ func regHTTPAddrFlag(cmd *kingpin.CmdClause) *string { return cmd.Flag("http-address", "Listen host:port for HTTP endpoints.").Default("0.0.0.0:10902").String() } -func modelDuration(flags *kingpin.FlagClause) *model.Duration { - value := new(model.Duration) - flags.SetValue(value) - - return value +func regHTTPGracePeriodFlag(cmd *kingpin.CmdClause) *model.Duration { + return modelDuration(cmd.Flag("http-grace-period", "Time to wait after an interrupt received for HTTP Server.").Default("5s")) } func regCommonObjStoreFlags(cmd *kingpin.CmdClause, suffix string, required bool, extraDesc ...string) *extflag.PathOrContent { diff --git a/cmd/thanos/main.go b/cmd/thanos/main.go index f232c9b5d6..3e93628e9b 100644 --- a/cmd/thanos/main.go +++ b/cmd/thanos/main.go @@ -8,9 +8,6 @@ import ( "io" "io/ioutil" "math" - "net" - "net/http" - "net/http/pprof" "os" "os/signal" "path/filepath" @@ -29,11 +26,7 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/version" - "github.com/thanos-io/thanos/pkg/component" - "github.com/thanos-io/thanos/pkg/prober" - "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/tracing" "github.com/thanos-io/thanos/pkg/tracing/client" @@ -230,18 +223,6 @@ func interrupt(logger log.Logger, cancel <-chan struct{}) error { } } -func registerProfile(mux *http.ServeMux) { - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) -} - -func registerMetrics(mux *http.ServeMux, g prometheus.Gatherer) { - mux.Handle("/metrics", promhttp.HandlerFor(g, promhttp.HandlerOpts{})) -} - func defaultGRPCTLSServerOpts(logger log.Logger, cert, key, clientCA string) ([]grpc.ServerOption, error) { opts := []grpc.ServerOption{} tlsCfg, err := defaultTLSServerOpts(log.With(logger, "protocol", "gRPC"), cert, key, clientCA) @@ -383,30 +364,3 @@ func newStoreGRPCServer(logger log.Logger, reg prometheus.Registerer, tracer ope return s } - -// scheduleHTTPServer starts a run.Group that servers HTTP endpoint with default endpoints providing Prometheus metrics, -// profiling and liveness/readiness probes. -func scheduleHTTPServer(g *run.Group, logger log.Logger, reg *prometheus.Registry, readinessProber *prober.Prober, httpBindAddr string, handler http.Handler, comp component.Component) error { - mux := http.NewServeMux() - registerMetrics(mux, reg) - registerProfile(mux) - readinessProber.RegisterInMux(mux) - if handler != nil { - mux.Handle("/", handler) - } - - l, err := net.Listen("tcp", httpBindAddr) - if err != nil { - return errors.Wrap(err, "listen metrics address") - } - - g.Add(func() error { - level.Info(logger).Log("msg", "listening for requests and metrics", "component", comp.String(), "address", httpBindAddr) - readinessProber.SetHealthy() - return errors.Wrapf(http.Serve(l, mux), "serve %s and metrics", comp.String()) - }, func(err error) { - readinessProber.SetNotHealthy(err) - runutil.CloseWithLogOnErr(logger, l, "%s and metric listener", comp.String()) - }) - return nil -} diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index e4c98de02b..c9d93b4feb 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -31,6 +31,7 @@ import ( "github.com/thanos-io/thanos/pkg/query" v1 "github.com/thanos-io/thanos/pkg/query/api" "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/server" "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/tracing" "github.com/thanos-io/thanos/pkg/ui" @@ -45,6 +46,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(comp.String(), "query node exposing PromQL enabled Query API with data retrieved from multiple store nodes") httpBindAddr := regHTTPAddrFlag(cmd) + httpGracePeriod := regHTTPGracePeriodFlag(cmd) grpcBindAddr, srvCert, srvKey, srvClientCA := regGRPCFlags(cmd) secure := cmd.Flag("grpc-client-tls-secure", "Use TLS when talking to the gRPC server").Default("false").Bool() @@ -140,6 +142,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { *caCert, *serverName, *httpBindAddr, + time.Duration(*httpGracePeriod), *webRoutePrefix, *webExternalPrefix, *webPrefixHeaderName, @@ -222,6 +225,7 @@ func runQuery( caCert string, serverName string, httpBindAddr string, + httpGracePeriod time.Duration, webRoutePrefix string, webExternalPrefix string, webPrefixHeaderName string, @@ -376,9 +380,13 @@ func runQuery( api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, router, comp); err != nil { - return errors.Wrap(err, "schedule HTTP server with probes") - } + srv := server.NewHTTP(logger, reg, comp, statusProber, + server.WithListen(httpBindAddr), + server.WithGracePeriod(httpGracePeriod), + ) + srv.Handle("/", router) + + g.Add(srv.ListenAndServe, srv.Shutdown) } // Start query (proxy) gRPC StoreAPI. { diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index a6858d3bf3..649b03ef5c 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -9,6 +9,7 @@ import ( "time" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/server" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -36,6 +37,7 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(comp.String(), "Accept Prometheus remote write API requests and write to local tsdb (EXPERIMENTAL, this may change drastically without notice)") httpBindAddr := regHTTPAddrFlag(cmd) + httpGracePeriod := regHTTPGracePeriodFlag(cmd) grpcBindAddr, grpcCert, grpcKey, grpcClientCA := regGRPCFlags(cmd) rwAddress := cmd.Flag("remote-write.address", "Address to listen on for remote write requests."). @@ -109,6 +111,7 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { *grpcKey, *grpcClientCA, *httpBindAddr, + time.Duration(*httpGracePeriod), *rwAddress, *rwServerCert, *rwServerKey, @@ -142,6 +145,7 @@ func runReceive( grpcKey string, grpcClientCA string, httpBindAddr string, + httpGracePeriod time.Duration, rwAddress string, rwServerCert string, rwServerKey string, @@ -335,9 +339,11 @@ func runReceive( level.Debug(logger).Log("msg", "setting up http server") // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, comp); err != nil { - return errors.Wrap(err, "schedule HTTP server with probes") - } + srv := server.NewHTTP(logger, reg, comp, statusProber, + server.WithListen(httpBindAddr), + server.WithGracePeriod(httpGracePeriod), + ) + g.Add(srv.ListenAndServe, srv.Shutdown) level.Debug(logger).Log("msg", "setting up grpc server") { diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 5e76f5bc1f..dedb979b2c 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -18,6 +18,7 @@ import ( "time" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/server" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -62,6 +63,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(comp.String(), "ruler evaluating Prometheus rules against given Query nodes, exposing Store API and storing old blocks in bucket") httpBindAddr := regHTTPAddrFlag(cmd) + httpGracePeriod := regHTTPGracePeriodFlag(cmd) grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) labelStrs := cmd.Flag("label", "Labels to be applied to all generated metrics (repeated). Similar to external labels for Prometheus, used to identify ruler and its blocks as unique source."). @@ -161,6 +163,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { *key, *clientCA, *httpBindAddr, + time.Duration(*httpGracePeriod), *webRoutePrefix, *webExternalPrefix, *webPrefixHeaderName, @@ -196,6 +199,7 @@ func runRule( key string, clientCA string, httpBindAddr string, + httpGracePeriod time.Duration, webRoutePrefix string, webExternalPrefix string, webPrefixHeaderName string, @@ -548,9 +552,13 @@ func runRule( api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, router, comp); err != nil { - return errors.Wrap(err, "schedule HTTP server with probes") - } + srv := server.NewHTTP(logger, reg, comp, statusProber, + server.WithListen(httpBindAddr), + server.WithGracePeriod(httpGracePeriod), + ) + srv.Handle("/", router) + + g.Add(srv.ListenAndServe, srv.Shutdown) } confContentYaml, err := objStoreConfig.Content() diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 416fb67d2d..e9668065a6 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -9,6 +9,7 @@ import ( "time" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/server" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -36,6 +37,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(component.Sidecar.String(), "sidecar for Prometheus server") httpBindAddr := regHTTPAddrFlag(cmd) + httpGracePeriod := regHTTPGracePeriodFlag(cmd) grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) promURL := cmd.Flag("prometheus.url", "URL at which to reach Prometheus's API. For better performance use local network."). @@ -81,6 +83,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { *key, *clientCA, *httpBindAddr, + time.Duration(*httpGracePeriod), *promURL, *promReadyTimeout, *dataDir, @@ -103,6 +106,7 @@ func runSidecar( key string, clientCA string, httpBindAddr string, + httpGracePeriod time.Duration, promURL *url.URL, promReadyTimeout time.Duration, dataDir string, @@ -134,11 +138,14 @@ func runSidecar( uploads = false } - statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, comp); err != nil { - return errors.Wrap(err, "schedule HTTP server with probes") - } + statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + srv := server.NewHTTP(logger, reg, comp, statusProber, + server.WithListen(httpBindAddr), + server.WithGracePeriod(httpGracePeriod), + ) + + g.Add(srv.ListenAndServe, srv.Shutdown) // Setup all the concurrent groups. { diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 6326b38104..54e8a63124 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -6,6 +6,7 @@ import ( "time" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/server" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -30,6 +31,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(component.Store.String(), "store node giving access to blocks in a bucket provider. Now supported GCS, S3, Azure, Swift and Tencent COS.") httpBindAddr := regHTTPAddrFlag(cmd) + httpGracePeriod := regHTTPGracePeriodFlag(cmd) grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). @@ -83,6 +85,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { *key, *clientCA, *httpBindAddr, + time.Duration(*httpGracePeriod), uint64(*indexCacheSize), uint64(*chunkPoolSize), uint64(*maxSampleCount), @@ -114,6 +117,7 @@ func runStore( key string, clientCA string, httpBindAddr string, + httpGracePeriod time.Duration, indexCacheSizeBytes uint64, chunkPoolSizeBytes uint64, maxSampleCount uint64, @@ -128,9 +132,12 @@ func runStore( ) error { // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) - if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, component); err != nil { - return errors.Wrap(err, "schedule HTTP server") - } + srv := server.NewHTTP(logger, reg, component, statusProber, + server.WithListen(httpBindAddr), + server.WithGracePeriod(httpGracePeriod), + ) + + g.Add(srv.ListenAndServe, srv.Shutdown) confContentYaml, err := objStoreConfig.Content() if err != nil { diff --git a/docs/components/compact.md b/docs/components/compact.md index b39b65ee65..df1056f571 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -84,6 +84,8 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. + --http-grace-period=5s Time to wait after an interrupt received for HTTP + Server. --data-dir="./data" Data directory in which to cache blocks and process compactions. --objstore.config-file= diff --git a/docs/components/query.md b/docs/components/query.md index c10c03b328..3ebf692471 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -259,6 +259,8 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. + --http-grace-period=5s Time to wait after an interrupt received for + HTTP Server. --grpc-address="0.0.0.0:10901" Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable diff --git a/docs/components/rule.md b/docs/components/rule.md index ba766090e0..a697dc02fb 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -168,6 +168,8 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. + --http-grace-period=5s Time to wait after an interrupt received for + HTTP Server. --grpc-address="0.0.0.0:10901" Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index a1d1a92112..5546ad20f2 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -101,6 +101,8 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. + --http-grace-period=5s Time to wait after an interrupt received for + HTTP Server. --grpc-address="0.0.0.0:10901" Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable diff --git a/docs/components/store.md b/docs/components/store.md index 9e5b7fe943..6f11cebc9d 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -51,6 +51,8 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. + --http-grace-period=5s Time to wait after an interrupt received for + HTTP Server. --grpc-address="0.0.0.0:10901" Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable diff --git a/pkg/server/http.go b/pkg/server/http.go new file mode 100644 index 0000000000..344ebac78f --- /dev/null +++ b/pkg/server/http.go @@ -0,0 +1,102 @@ +package server + +import ( + "context" + "net" + "net/http" + "net/http/pprof" + "time" + + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/prober" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type Server struct { + logger log.Logger + comp component.Component + prober *prober.Prober + + mux *http.ServeMux + srv *http.Server + listener net.Listener + + opts options +} + +func NewHTTP(logger log.Logger, reg *prometheus.Registry, comp component.Component, prober *prober.Prober, opts ...Option) Server { + options := options{ + gracePeriod: 5 * time.Second, + listen: "0.0.0.0:10902", + } + + for _, o := range opts { + o.apply(&options) + } + + mux := http.NewServeMux() + registerMetrics(mux, reg) + registerProfiler(mux) + prober.RegisterInMux(mux) + + return Server{ + logger: log.With(logger, "service", "http/server"), + comp: comp, + prober: prober, + mux: mux, + srv: &http.Server{Addr: options.listen, Handler: mux}, + opts: options, + } +} + +func (s *Server) ListenAndServe() error { + l, err := net.Listen("tcp", s.opts.listen) + if err != nil { + return errors.Wrap(err, "listen metrics address") + } + s.listener = l + s.prober.SetHealthy() + level.Info(s.logger).Log("msg", "listening for requests and metrics", "component", s.comp.String(), "address", s.opts.listen) + + return errors.Wrapf(http.Serve(l, s.mux), "serve %s and metrics", s.comp.String()) +} + +func (s *Server) Shutdown(err error) { + s.prober.SetNotReady(err) + defer s.prober.SetNotHealthy(err) + + if err == http.ErrServerClosed { + level.Warn(s.logger).Log("msg", "internal server closed unexpectedly") + return + } + + ctx, cancel := context.WithTimeout(context.Background(), s.opts.gracePeriod) + defer cancel() + + level.Info(s.logger).Log("msg", "server shut down internal server") + + if err := s.srv.Shutdown(ctx); err != nil { + level.Error(s.logger).Log("msg", "server shut down failed", "err", err, "component", s.comp.String()) + } +} + +func (s *Server) Handle(pattern string, handler http.Handler) { + s.mux.Handle(pattern, handler) +} + +func registerProfiler(mux *http.ServeMux) { + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) +} + +func registerMetrics(mux *http.ServeMux, g prometheus.Gatherer) { + mux.Handle("/metrics", promhttp.HandlerFor(g, promhttp.HandlerOpts{})) +} diff --git a/pkg/server/option.go b/pkg/server/option.go new file mode 100644 index 0000000000..702b6dbecc --- /dev/null +++ b/pkg/server/option.go @@ -0,0 +1,33 @@ +package server + +import ( + "time" +) + +type options struct { + gracePeriod time.Duration + listen string +} + +// Option overrides behavior of Server. +type Option interface { + apply(*options) +} + +type optionFunc func(*options) + +func (f optionFunc) apply(o *options) { + f(o) +} + +func WithGracePeriod(t time.Duration) Option { + return optionFunc(func(o *options) { + o.gracePeriod = t + }) +} + +func WithListen(s string) Option { + return optionFunc(func(o *options) { + o.listen = s + }) +} From e87f043b8572f724ee631d35eaab5bbeedf28ee4 Mon Sep 17 00:00:00 2001 From: obiesmans Date: Thu, 24 Oct 2019 18:51:08 +0200 Subject: [PATCH 021/257] Upload blocks with the lowest minTime first (#1679) Fixes #1670 Signed-off-by: Olivier Biesmans --- CHANGELOG.md | 1 + pkg/shipper/shipper.go | 13 ++++- pkg/shipper/shipper_test.go | 95 +++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b40e93880e..b17c57819b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1656](https://github.com/thanos-io/thanos/pull/1656) Thanos Store now starts metric and status probe HTTP server earlier in its start-up sequence. `/-/healthy` endpoint now starts to respond with success earlier. `/metrics` endpoint starts serving metrics earlier as well. Make sure to point your readiness probes to the `/-/ready` endpoint rather than `/metrics`. - [#1669](https://github.com/thanos-io/thanos/pull/1669) Fixed store sharding. Now it does not load excluded meta.jsons and load/fetch index-cache.json files. +- [#1670](https://github.com/thanos-io/thanos/pull/1670) Fixed un-ordered blocks upload. Sidecar now uploads the oldest blocks first. ### Changed diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index 6631c274f7..3f4267e95d 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -380,10 +380,12 @@ func (s *Shipper) upload(ctx context.Context, meta *metadata.Meta) error { return block.Upload(ctx, s.logger, s.bucket, updir) } -// iterBlockMetas calls f with the block meta for each block found in dir. It logs -// an error and continues if it cannot access a meta.json file. +// iterBlockMetas calls f with the block meta for each block found in dir +// sorted by minTime asc. It logs an error and continues if it cannot access a +// meta.json file. // If f returns an error, the function returns with the same error. func (s *Shipper) iterBlockMetas(f func(m *metadata.Meta) error) error { + var metas []*metadata.Meta names, err := fileutil.ReadDir(s.dir) if err != nil { return errors.Wrap(err, "read dir") @@ -407,6 +409,13 @@ func (s *Shipper) iterBlockMetas(f func(m *metadata.Meta) error) error { level.Warn(s.logger).Log("msg", "reading meta file failed", "err", err) continue } + metas = append(metas, m) + } + sort.Slice(metas, func(i, j int) bool { + return metas[i].BlockMeta.MinTime < metas[j].BlockMeta.MinTime + }) + for _, m := range metas { + if err := f(m); err != nil { return err } diff --git a/pkg/shipper/shipper_test.go b/pkg/shipper/shipper_test.go index a0f02572e8..85a78b16dc 100644 --- a/pkg/shipper/shipper_test.go +++ b/pkg/shipper/shipper_test.go @@ -3,8 +3,10 @@ package shipper import ( "io/ioutil" "math" + "math/rand" "os" "path" + "sort" "testing" "github.com/go-kit/kit/log" @@ -77,3 +79,96 @@ func TestShipperTimestamps(t *testing.T) { testutil.Equals(t, int64(1000), mint) testutil.Equals(t, int64(2000), maxt) } + +func TestIterBlockMetas(t *testing.T) { + var metas []*metadata.Meta + dir, err := ioutil.TempDir("", "shipper-test") + testutil.Ok(t, err) + defer func() { + testutil.Ok(t, os.RemoveAll(dir)) + }() + + id1 := ulid.MustNew(1, nil) + testutil.Ok(t, os.Mkdir(path.Join(dir, id1.String()), os.ModePerm)) + testutil.Ok(t, metadata.Write(log.NewNopLogger(), path.Join(dir, id1.String()), &metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ + ULID: id1, + MaxTime: 2000, + MinTime: 1000, + Version: 1, + }, + })) + + id2 := ulid.MustNew(2, nil) + testutil.Ok(t, os.Mkdir(path.Join(dir, id2.String()), os.ModePerm)) + testutil.Ok(t, metadata.Write(log.NewNopLogger(), path.Join(dir, id2.String()), &metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ + ULID: id2, + MaxTime: 5000, + MinTime: 4000, + Version: 1, + }, + })) + + id3 := ulid.MustNew(3, nil) + testutil.Ok(t, os.Mkdir(path.Join(dir, id3.String()), os.ModePerm)) + testutil.Ok(t, metadata.Write(log.NewNopLogger(), path.Join(dir, id3.String()), &metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ + ULID: id3, + MaxTime: 3000, + MinTime: 2000, + Version: 1, + }, + })) + + shipper := New(nil, nil, dir, nil, nil, metadata.TestSource) + if err := shipper.iterBlockMetas(func(m *metadata.Meta) error { + metas = append(metas, m) + return nil + }); err != nil { + testutil.Ok(t, err) + } + testutil.Equals(t, sort.SliceIsSorted(metas, func(i, j int) bool { + return metas[i].BlockMeta.MinTime < metas[j].BlockMeta.MinTime + }), true) +} + +func BenchmarkIterBlockMetas(b *testing.B) { + var metas []*metadata.Meta + dir, err := ioutil.TempDir("", "shipper-test") + testutil.Ok(b, err) + defer func() { + testutil.Ok(b, os.RemoveAll(dir)) + }() + + for i := 0; i < 100; i++ { + id := ulid.MustNew(uint64(i), nil) + testutil.Ok(b, os.Mkdir(path.Join(dir, id.String()), os.ModePerm)) + testutil.Ok(b, + metadata.Write( + log.NewNopLogger(), + path.Join(dir, id.String()), + &metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ + ULID: id, + MaxTime: int64((i + 1) * 1000), + MinTime: int64(i * 1000), + Version: 1, + }, + }, + ), + ) + } + rand.Shuffle(len(metas), func(i, j int) { + metas[i], metas[j] = metas[j], metas[i] + }) + b.ResetTimer() + + shipper := New(nil, nil, dir, nil, nil, metadata.TestSource) + if err := shipper.iterBlockMetas(func(m *metadata.Meta) error { + metas = append(metas, m) + return nil + }); err != nil { + testutil.Ok(b, err) + } +} From f551b8eb8ff793c0fbb89825e2c1f2d4c081ae1d Mon Sep 17 00:00:00 2001 From: mengskysama Date: Fri, 25 Oct 2019 16:13:09 +0800 Subject: [PATCH 022/257] fix: oss range in 32bit os (#1682) Signed-off-by: mengskysama --- pkg/objstore/oss/oss.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/objstore/oss/oss.go b/pkg/objstore/oss/oss.go index bd913f6c96..95fe147b45 100644 --- a/pkg/objstore/oss/oss.go +++ b/pkg/objstore/oss/oss.go @@ -269,7 +269,7 @@ func (b *Bucket) setRange(start, end int64, name string) (alioss.Option, error) return nil, err } - size, err := strconv.ParseInt(header["Content-Length"][0], 10, 0) + size, err := strconv.ParseInt(header["Content-Length"][0], 10, 64) if err != nil { return nil, err } From 95da4909f89b750b83e7da2e13f65310bcb92d5e Mon Sep 17 00:00:00 2001 From: Jinhu Wu Date: Fri, 25 Oct 2019 17:16:20 +0800 Subject: [PATCH 023/257] fix docs (#1683) Signed-off-by: wujinhu --- CHANGELOG.md | 2 +- docs/storage.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b17c57819b..059eddbcc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Added - [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. -- [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss-configuration) for further information. +- [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss) for further information. - [#1680](https://github.com/thanos-io/thanos/pull/1680) Add a new `--http-grace-period` CLI option to components which serve HTTP to set how long to wait until HTTP Server shuts down. ### Fixed diff --git a/docs/storage.md b/docs/storage.md index 2abd2b5ea1..27f4499af3 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -338,7 +338,7 @@ config: Set the flags `--objstore.config-file` to reference to the configuration file. -## AliYun OSS Configuration +## AliYun OSS In order to use AliYun OSS object storage, you should first create a bucket with proper Storage Class , ACLs and get the access key on the AliYun cloud. Go to [https://www.alibabacloud.com/product/oss](https://www.alibabacloud.com/product/oss) for more detail. To use AliYun OSS object storage, please specify following yaml configuration file in `objstore.config*` flag. From 9299aa65acd5f16fca07543ded5c383e32605f4a Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 25 Oct 2019 14:30:05 +0100 Subject: [PATCH 024/257] Bumped version file. (#1685) Kind of annoying and easy to forget: it should be done by promu eventually. Will investigate later. Signed-off-by: Bartek Plotka --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f8d71478f5..d182dc9160 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8.0-dev +0.8.1-dev From 7b4d0601de5c9bd5b923fa557e2573b5ea4a6355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicent=20Mart=C3=AD?= Date: Mon, 28 Oct 2019 16:03:04 +0100 Subject: [PATCH 025/257] http: shutdown the actual http server (#1689) PR #1680 introduced graceful handling for the HTTP server in Thanos, but the graceful `Shutdown` call was being performed on an `http.Server` instance that was *not* running at all. The actual server that was listening for requests was started through `http.Serve`, so there was no reference to the server struct that we could use to shut it down. This was causing all of Thanos to freeze after receiving an exit signal, because the run-group for the HTTP server would never finalize. This seems like an oversight because the `(*Server).srv` field was being properly initialized with an HTTP server. Fix this by calling `ListenAndServe` on our initialized server. Signed-off-by: Vicent Marti --- pkg/server/http.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pkg/server/http.go b/pkg/server/http.go index 344ebac78f..d7b17a5a77 100644 --- a/pkg/server/http.go +++ b/pkg/server/http.go @@ -2,7 +2,6 @@ package server import ( "context" - "net" "net/http" "net/http/pprof" "time" @@ -22,9 +21,8 @@ type Server struct { comp component.Component prober *prober.Prober - mux *http.ServeMux - srv *http.Server - listener net.Listener + mux *http.ServeMux + srv *http.Server opts options } @@ -55,15 +53,9 @@ func NewHTTP(logger log.Logger, reg *prometheus.Registry, comp component.Compone } func (s *Server) ListenAndServe() error { - l, err := net.Listen("tcp", s.opts.listen) - if err != nil { - return errors.Wrap(err, "listen metrics address") - } - s.listener = l s.prober.SetHealthy() level.Info(s.logger).Log("msg", "listening for requests and metrics", "component", s.comp.String(), "address", s.opts.listen) - - return errors.Wrapf(http.Serve(l, s.mux), "serve %s and metrics", s.comp.String()) + return errors.Wrapf(s.srv.ListenAndServe(), "serve %s and metrics", s.comp.String()) } func (s *Server) Shutdown(err error) { From 0e6b8aababd7a994a52a45f8ec28b8e3f187f2f2 Mon Sep 17 00:00:00 2001 From: obiesmans Date: Mon, 28 Oct 2019 17:37:26 +0100 Subject: [PATCH 026/257] Add BlaBlaCar as an adopter (#1684) Signed-off-by: Olivier Biesmans --- website/data/adopters.yml | 3 +++ website/static/logos/blablacar.png | Bin 0 -> 30235 bytes 2 files changed, 3 insertions(+) create mode 100644 website/static/logos/blablacar.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index d19cf0761b..8eac02bc75 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -54,3 +54,6 @@ adopters: - name: Softonic url: https://www.softonic.com logo: softonic.png +- name: BlaBlaCar + url: https://www.blablacar.com + logo: blablacar.png diff --git a/website/static/logos/blablacar.png b/website/static/logos/blablacar.png new file mode 100644 index 0000000000000000000000000000000000000000..05ad56e9dcb6110b4c414d3601c517a3181aa139 GIT binary patch literal 30235 zcmZ6y2|QHa8$Ui8U7~9#NiLdHLiRLcXQWaIrBI5oi(%|0B)XJPDT!1}QB;<(hLEvj z$-XsX8Dl51uciDieZIfn|Nrmxs_xwToacEz?`JvZo;h>h*zh#SCCr6Fp};fxdKXbB zJ}e4_R%b^ecdkzsJ0SnDJL1pdQK;-k-0Rlck}$e)EaEqMU;K|Iv*M)!4kNX5@Lp9{>l z@|(v0Uts=ux^)+ERf|hQ{{PUX#br=& zenj@IoRe_c`xRi3+Z zDPZh3qXwQYwbK4O_-G$WuHd6KIo?mhJZ}H`he8x8POU4op{UT^B=ODWr}x^5{eu

U$Rm>+nnu8bPb2bDa;S2v?#?e>Wl3g;nlH)7Lv6qGz2tZk z4f-=~N#t?6>5}e(l$=Pty}(`+YsZOF`Zc-T->Nr1aJ={>;#Fu+BHR2L8P~7g<>){6 zWz+l_4>nZUh0`kiUxvxMa9KB!`=inpIBKBsr@@M1 z14<#Pd{A^!Szca@_)R&28zs~I!EmL*|AC*qQuTJ!GnX_okFwjUa|El*feRjN!S>x9 zE;q>s=2!*SA`vIg<$W|;6p!G>jYKf-bD(4zvn|awzE!yvNpB9eznqon=P_WMc^((B^O820i9gq-y zsUFn&iikoDWz5^dw+yi4p==bD^F%R`&nv<_Vsp%tO%842*R57H31JQYqbQjZEmDK; z`}b~tCI0uj1sh?XGa~C9vAUnMJ@_Q28ZIsA*R3{7q^mGkMi)}UzXZsjWOQ4kY-+Al zQB3r5!%;F-ir2C9dH|(>H&z~3c`o7|VXj3+r0^X@_?n4iL@K5ZRSPUM0C?>RC#8MC zVf44(02c-dB@wQ6atilc#3y3&_DSi8M6oWq8S`t+XRz+Y*| ze_jGj<^1f;z)T6W+9{zT-3{WvH-`L;;@ha4)~VrtBw~0s#rOI~&`CI-`VFPEcku>5 z@SJn;L~{97NQOVv|8HD$f3zH=mne=zad_{|2)d)vTEUj|!-*XHu+*D6JCq%~c3y&H zkC>C67szgx8GOAd(2CH7ncuQE;o4>o*P2w#Ga}hApKB9zlj??R{~7wHQuRN%utf3Y zI^gk^L;CWI0@=+ngZ~oj#l{Y)O)g4WOseE^!zXQ&_MyJ2BV1=8cH=RN0&^Whk<$?f z8X?(&ceC$njv%^_7-BvjxQ-3iJgNMAr;aa@AuLlYDtL#k5&vfciF<5QLhz|A%zf%DLRh?xFnv4)WfjvL=NrhrYCod_|*+0 zzQ}EHlYn5{@(EKko>1}r&yk;7K`DIKVlAjh{-<&?6yphZ5R9Ii^wA>gMsiQ1gNX_s z>xNm?c1S%A3*L<Pvv+z9S%}dN;$kb%|mSEOWzKk8fw7f_o7F_{A+(5jBW^nVS$1 zD$V}YyCwPjK*xFv4~?;zsqe$S3uqnRNcm^?Vo%C9pf$2?lU7%Rko2tx{u0RkW9R98 zo`%R5ZC1M>;@(_^3)jyhF}w&xgp%0=-QQQL#5b#Hvwljp;B{;QA)kLuK8~^zv@~(j zdcLlvCcH^k{;Wxr$)r42))slWm&=HJmL>{F8aNCb4_nR+Ke?$Ni;NE3vibaHTch~b z?*dGJEb5Ri(myuCVZbK_CKyTF`z}z=(F_Ea47zYNraoHUA{!}6&PTwe6GS98dbhe< z;Q7}}cAoz!6m7SMGmw6}yXw1u;e#)dW$##H0w{f%vUe?kf{N?k1(0?Y&4`wVs{Um} zHWJ?!%DY*{XEsdMYJ%=$-SFyGpp+uBr1BCCf<%)*Eiw)@d>K*v@5wF~V-qLMNOClY zB9v@)i7i0->G<=n=VWZ)ggaOCP`i5m*HWE~O;qy#ym_%oL;{OM^Hlq3(0MDnKOzHs zu*maGek@Ov5vg$?ie>Kz<6Jk6q~OEmGVY;ut|H|_*y>ZDAwOg!flHHGRo#wM|Gq|X zH^>fN5#vE!$xo9~ZUR8*s7+#AknSVgArNtIGrwNc4GSLMzL{V7NaGb=Z+n8|;I7|} zFwx!l*GbubKqlke1U-&!_ClolIbGPob5&M4YUx(Hky`vo_Tagz9Rk8e5_^#;;;YLA z6DRfOy>F02SmRxuW_@x!7>Rbbw`lD&4mJMLse zNy^d+lB#wu?rdIfD%35IDpB-A)k=#CR}~mM{AVg?p`7izb3 z)cz1-TP&XQp`nN^cTjoP-=Gl8T2!7>0!4aI&g>;n1NVZlx z!nFsv7!_%V@RjPo8&Y{o7i#04Va>kspj6K<*u&AXQCByola42#O@V^jPEpj82u6hv zLrK+P%VoRcMsb@@5Z-i56?+7xw>=vFPqA6ZX@T6Z*3BxAcHp*k!iN8|bEcYT;*_ji>`iQKCpjt6i1gaJ4Q`@I~oM zIdR+CeH(T`0;7Xng-;<}qi?&bbrdmDXg3&W$QGR9fb@`-f01)eMEbj{b%!8-aLSzm zDfR*bttTLgvQBj0>!*jH%SZasd+hbe;-sHQvWwz9ZU?>-t&K!US%#_xP&cKBBP?g548aPCzyvQPfe4J3SnTJxQD*7*f4L? zT)9GIW8nWK@fj?qG~|TD#qffK&`iZcN$0JI4J#Xc7o5{WJkv}^7(rD03-DvGDIY00 zP4<&;`Lnw3TN(MG6i@gzoVE;T3u&V4lpB$Vq?Lvs_-g?lJ2*{V*=19GF4DivH}y5N z=bwynCT%jWnt|v(h&U)+3;0=|7U*P_BZ*M^elt%$Y^g|5u0xh>-CGFHwjk_c%BAdSt)-X-$+>RtH9{oDIprxD_Gripx=|CA-v1BkV# z$6&=(iF^54ZAex=-OLvA$%C79E|tF2dcA}HRV{#`-lNJ!W&K^3PfGmb);7i;map|| z(2Fa^=(M-5vlANz@@@oTayy-urj1C_L95A$)2U-~vN1;-o82sB1Ya1D@JyzhU>vTw zX{Q3qJusd-sjpWi$nWODJY!{)CY=A8!LpSplz)(Jnzh2iiSWlzo@uwIU=Oyh?dk+IiL78sZHXrYGWyUwoTYwpIEA18l&piz1P&-OO{c2QUzum3}cY+ufc zt2s15F?ZclaE9aiaXa{j)z^K0GagodyE900d?0i2_G#PIoF4l#CpJtJR?}OhcyBVv z24}_ocAxd`Z!PTY7%2ErwJvG*^KI1U+P$KgO3e-jT8pDotDEnHJjx_lZ;XU$zg*h? zs(IJ>9%pY^QRNp`JJ!2tORt}r{r#|HKia)8PY9xqSuy)HDtu*x7uWGUM^i6h=`Pl< z+hf%k9m#`LB4)4l<&Lh#CepPJCGuCiSxTE=W!(uOwk!=V-Jbj6Z5nyZ#ii}7x3LK} ztvEOm7vI_Iw&G7gXBp9#mtw&WvZA47ogUDJxyOuI zp4_fxPR8kt;Wf>3BRRCnv6=VxXB8TH>NU*O(qy2H;*V;p51Q$FaoX%~zy|L5B+wd` z+tM>vO4Gp#IcmP1zORtwD)4g8IS0`|2CpZya5m z?!0x{yY#*Bt&zMDPR9v*DUNq;##Y#AZ^HEX6@QIYM^{<+wWn0c|5bn`|R6t zJ0^Vg_`tseqtrTg+rz&#mY#a+yz_B@ms7L$Uo^4DrnzOfzU0IPwq~0!&h$~r2EIH6 zmlwq)B-C1SXhD-)_L?gq%&R^6BhP8ynqRp+_yKf+8*Z*fc5qfwdD>A+m1ywp&4V5} z@3A76^1HCzZHPu${3jx1>**8FdGjfMmusWtPYsTzKS(*W!U3H=%nj@_ z(7Cuf49%cyxX-5jAR0GEs(gFth`_Jg$72U;Uc1K|n)J$>L78!PFXiWHbA$(e9Q^^D zG3&I0`!q$ZF(p3Tr@IOm75JYF$}@KbH>{A1NE)O(^p|>~T8=?*eAEbYI9cb$^aKEV&+?cTIa^U8W%E5yJg!TLv zcuRXzoY0q9Y^{q^4$ZC7x?!^S$P0T|tde#nf@n+=xm+F~dZPD42lzzpLmi;LZ>7`C zmFMd*l2<5<__|(I(r}aa0#0%u>=j#19FZDXvtvm6Rl1iQyxebL$hIdGZrkm}{~P}+ zslF3@@-T=&;5f2xNSh-T?Ed~mwDsl_qH&+^*}?;+)r>(rc(>}ajKdo#`tT`nLR z`~Uaksp7AI+m=tkA0=NSp z(Wqm*hw6ci{&Z?X3|P;sGq1fr{X1K?kMi{RaR$MzCM=3QOC-v=+kuPYi^#Z3q z#lmSiTvCj`a;~xTtpbCRyE~Ucj8=jUUArtfr$ zQj4TuCD7@DZc?lOwh>zhm)Wq?+DS6(&eVC=uoH@kIN&R&Cn8jR?lZ9kBf@^PpxPGw z`22PzhS{)p!ss42Jbm$}XY{K3y_RwQucjZlvgb|9i&Lup?jrG##tJ`KEcrXa2mE)H zSohrp&oU^Q9(<9{(K$3UlX8ZH@%(HJ19@zN<&r!v9AKatpVsHBgN>$J4T9U;M3Wkx zl{PU5;YsZL98|m_q)dNU$f+FQ>pBXS;FCv`>8v4@uZg=fscb*P`?5pI5t|Bms&m$` z#HPwa4}~4k-TVQz=nyx(61=`G00~v$_oznui3sWCW*NK_%{IeI_BRiKx^7~mD8aH! z1|jX>4so8vp)wyV>t}drHeOYI02I!6M+01)&{b?i3=IGa4psogMWQ4=uN6*+0ty|X z>qLPnHFO9P=odp^RqVvBVn#cMb+l?Yp{Kccc#yybISKBW1r$F$sutgeO`8ufbyk&N zK8jmrQ1B|`L$OX0ekJ&po4klRJ`PhTwCj;VK^<^`qACe#u)u^xpjrihO7>e6XNG?n zqHg&LqIT^x(Z1AJfEZ=EM3Wc%OiibXK3aS?so( z9!f+^IS>w)(JAhXuaMrW<**eIEIE#Iq-Jt)#>wwCJJiwm?|JmgSpZ*(zkwDe8VC8E zvZeKPdagBxCdinX`K6vnNqtGyeyC=RuELuUjp;29aZVivD}kA?-uhDQjd}koq?@;n z)^czQx>q~>_2LSLZrHoHYy=B?NNs3~J7Z6%r$wms`W0SSKS9J*Vm& zNX<^~Bk}om#mRC|Y3kXr7XZg^k6IV5zS{`4jY{z7=Y}vm#WrwGdI*kl60ARa1Cg?o zQ_^xgSl#pktK%Zg2DNy!K+w}7#}e8K_BAh-tlqV&Gc?rV2%f_m3EMVeu*!G@7=c|P z!>=39@Iob`BPYr>5|$&6jA$1Tc9=!R1<_F+tQw#{^T}ZGdY(xu@+J~>9c=r$^dmEn z5E;G`OFmYw&fgrf_s7KHzHn;D0{^M15P8UMz5QcO&|C49D4bBo^t`u^W+8q*an?b= zPdun`t@H8$>3KK3QjwO&I31DIY<%LcqaXC2(NC3xxH-%1ql=HjAoJRevy(fX<6QkM z)|S&4DpPe(UJWokQt)ctN6_x!lk1vHg0W>9_c+DwcraK$-Ov*qpLE|aA}C(|c;#l{ zVnoQ=`y!5qU7RrYbeRQyoyC3TcTFzawpVLj9 z{4|q(uQvz+e;PCzuRT1=b%Tm;jk>)XBsxIwcE~9k5BG&PO~#brYU3Vq|1`LdlO%5c ztlTeE0g$S?`N4U`$8YY~h%8kDKkuU*-#W0spsyM(Ru`z=Lm&_{;b#NhqS7Em@*6zA zi18)k2sf?NV3oA*kezVzgXNh4Fyc}4E1?qzPr$9;n=-uyoOH@duzNR08AqUN30_P@ zke49&qy#Z#VCJa38mb{8GNrRLGU4^o!oxt7hSh3s63TwSsyA^V<-z0loH zG*<4dc=h^<%GnES*cNMexhiDx_oMU@e3$T(jbt)RlaaT&DhE(Q+|LZ$wALDvlr9IP ziRFxM7nP03H{D8I0QO~cKqE3? zIKH_elM_hr?9OrJ_(_{sAx_?po2Db_8tV@)x>hI88x4SVYSv@EEgaBrf2`Ty$JsEE z(-?s_m$9pJ$?6`g!rS?aK;83$w+AYqozJc%e8@|BYDQyrCG;Y`T4=paqaynQDPr9+-c`=|5}UTO7HEsq-Px zhq(O~HAL_H={aoU(R0C6ya}--QDxT4`5hB;Mb&{cH3xiEFQ7Te27|+AxsryiH8|*& zBfKyRG)i##vCU9lKMe|BqKOG1xophiP%o}M=A0E7r8+yY z3!+%Qqf#>B*PCo~IcqM&5p{4j8BELt5sk^eJDEDu>r)}%A`RHsKS}6${PIIZV`wnQ zP7ijgaxTSRt-&wXhv5|NwH@+^{uynJe*NUmpxdFb^lhm^)ybcl71|z{7vc1k9~Nmx z;*eL$8-s+b-$$=!(`0%b4h>y4C=uqz27@-5&?pn*8t3&oL4FRn(^Hd&J5mYQ%t0tkRkiGX)c_mt@qYC#-Wo$~8zj!>RQ9 zC*3=9YK6$?-#H7t+kO#XB`e#!T_l&f+f=+4vSX`aUTA;`AVojLdG;vG0?SOy`>Rg_ zCaqy{UPm7pX;E`0m0*wn8*|5&;`Glt6n)-N5>hGs_~L3%FxXo#F1X?-I-8&t8E46$ zl)kyse3VC|gjU6R#{xx4@MmLH%{Xbm;exMC{i%`gy&Y_jK9wy?B@Mwo<-Ppjn*dYi znZDnzZ@Y=cD7$B$97Xxeiz5K-*7alytozOCkb5|NDQOn?V-Ap-q~-qTod^e&3>6a6 zD<%emM?|pEbHL9%d#)9GXdt$_YwdQGAS2=@jC|>s{!Y5D=8;HbIf4f~=vg=?W078V zLs{-M_efpXsQh9jFyXkp@fqvN_6c6B z(-Tlq`qCsQfv2NLm?_ZVhnFa}HXulIx)D1SO9LL{vYiaOT8h{0B6nO%nFZQj9xU#auwplmQx+))?wl5qlJ1xS zN4wXykyt|@PaZN?)P@p81qBZ873MiKl$BVDSaB|;T~d!500X-sREY%P6XSg~%5(FA zySHKa<`4tUOTds;&HP|OFE{gkHD{7s-k=XsYV1xFq4q^ zvT+tLoG{;JX8x%ua(2)R5~bW!GwH3{s4o`>_KlgWIuAGU@xRj*t|XcP%C5 zE|Ae4Bn2_VB`m4M4qXzW5&7_LnMQXgs3c~YWmuK6 zrgKc3h*|!TjAamZA1l>Qjcv6^ELnmO2EeabX4H^98$~|i5}&~Y{*hfeE6m_Gx^I~* z6LxGlRBQC}*8lc_GL> zBHNIppk|8{)jW{Q!0s%{9bNbC9qMYHv8=0 z3v0K{rdjn5!qm1+FYtprA3map(VzDFSq2y;noLH8C?OP^h8xlML9RaLDM4oBY zJvGc-cx&RZt@B&}EIMD_#g#+KV)sc}o+XE+W~^Bmb<~gs7k<@h^9-Djg={r+X%zIO zhMY;s7xx!K#b5S(b-vn)j5bs>{Q-W>VPjI1NYpUUz(&;f%Y{pS%#kTqrxt2)4@T5h z%^}_YJTkD@aLGq@FBepfK=xh%YF?U@BiGm$9*&o_A3gt_@GyWLd>1}jO2eZNZ&~1n zurDc5^OqX)J&oCrPONKO_5v5a8Hup>SW}FZq~Qx5IwoUJ>&}QzmAM9 z!3&WTo`JJ~#3xm~d;My|Wt_-pCzW!jaOT~!ZzG*Et2x&`j5#BHv;6P6QEt+C=aY*; zF69~$UkzN7&b(rL^1n5Wv;@IDQ&qxD9c^?*=8WM+{w3bjvHr|Ud!NiQDM48C%uLcx zj%B|?`QloaY_c1^x)m1}t84z*E0L7ge_i%SXH|2_e9ctPMuI>0rdO6Fc;+wBYG~a* zsDIz+0_kcICyAm-^_$r_HBq!I9@7;uZfSeQNu`Bj`F6&8hvt;=Rja8=nx7mDfWrp!ie@dZ|Aoe^I%Y8=&~`TejordL%Djq(*OSf^?O3H%-@Xfy zE7_Qs54hFYwLe~ATD$wxLRWy;9Q)&U557ZHS)qgOMU1epF zylHU0tt1zx-=NF&9C7Oy^~8KkV8Lvln{6JHa&5*dw>c$hO(>53j*QlN-%^Z2D*P}> z;d7(#yjQCt?+mD%+ITX4WW>y=5EeLff3`k2f1z_!D~65w|36EaO+K6Zn5SqZKe_1 z_vAG$l1?Q)pQ&S}`THgu^P14H;<>B65h7lGUG)geJO?aap&K44RlK=77~DlT6*Oyl zvU%97-8$e&aJwdxVvQy9d<7x}89G`wMrUNhxVZ;P@%BW8o;^J2Q-6<5#Ord(YZbmW z2&s*cTLup&A3ukAS$BYlm>gmftYUGJ-IUr7XdKY@{x5Ym^zte1)I!*w_TBILFe9SP z6-0&IWQnj2T3Y&|q~dmKG&8891V=xr6*j2;^xWged!-B!%m9NRtwep|sa9=MwVE?P zZa-jQsL6j*qIqhX7E{&F2F)%f`J{=XO7Sfs4m}1uD=-(Hv4W z0*Nq{)}^sf_a+1-l78%p+%298pyW@QC5iGm)I0#*JvXr7>~*`^>|wS=_jysGyOd_8 zxcjWs0LOl0V90T(fny^3_tX$L+HVc-LbeanW`!calv0}VHi$U*1}`O2coQz5v&^jt z+vVI|%zEMQbb?!8^|7$mzlr5Q|KY!U^k4^M-54^z%VrWZydl)XM?q?TU5h=8mds5yB2i6G=^0%Wmt6t8lo}wJy^gR7cJ|Z$kr6OW*6!3Z;Pbqa z*0zf`;fJd=8Mm-HidM=Z?qb%x4OF(92lNQXFZ)Apc8Y|OVFjNNBI^Zz?h?R347jui z6e2Yg1i)lk^XL_bA1$#%>uZ@<)4fUzi61!X!DT z4%{zc^O0J&djZb=IqV?6!?tM12^my;&{=$2FC=2263@igT=`atA0WCbR{^e;vU)Yp zB`$kOYC!5d(D8PbsSG(OSgk%StMF5-#WQtPoiF9FU#nRJ_$OnVfUQ zK-!w`>*@O?H1_E$YD8+<&MA0XQPf>JrB!9Ztn%&iG0>(rHaCQHs-Y92ScFI5oSqJ1 zh9wZ*(S_?DsWkiz!fA5vmF@_A#{@=@;j#j@kUd>Qbu&;lC2YlEPGtD((Vcg8ItY`D zU!Smm!MtDR#v8K+Hc)Dm2WF&Ld6gqqm>4c&zYPy0jU*f=K$fgtB_!kQK^=%lnXP)# zZ3=Ig!XS_z0$N-iAHVIcR1I`Q?dW$pex>QmGhFzKn}s;ZSD?T8M8XyB3$sAJ4TgQO zSb6t1^~HW^Dn3<(DqG05{Ekl0-F-e;#rv3Kj_Hni4-$)XzNrK99v8fW%qR|+*_jjK zB%KVW7szQT7oMrR4UEW)8iDa)p{pw$rUKi{_6RXCPds0O>-aE(Qrev|HZn=!{i$tM zTB&nD$6#RIHz4vyc~kj|!0g4Zb`PDo&;K^`88?hyusHmtg;Bo$Qf&uRUcI9iQm%hp zUb=CkgzaWiAF`njuL)!s?BUZRzofOsIlp~04chFLd9`UpqZE%O9dIWpNS>|$D&((_ zf^Q>>Nl^Nyz-^rZ9g9a&l>Vn>lDvKUDu9oAHxceQ_RKE=9X>U)qYst=j>*^2U!mx| zSFe5~D!k{F`E9E3G!}8F+{*l8=RLPmRNBcmB`VM=%yrWr-pi~FdmwxG zKwbGm>bT2;Y_>Vz-tetByK(<=)w97Dom6;gq8j+pzF(%!V_MkBksdv>NLJSRAxal6W|a|bzF(OI4& z;=ldZ^;gOMw@Z8j^U?y1WOL{3Na?Qq2tq?Q|bd=wl#H4XWu%qF$nr9)cC_0bEb5Ryt%+ zY9u$>P54&5JE1|ob0Frz>eWbu9}&8kXX4KX6nw1bIMxvsp_hLtu0BiP zj9|@d=Qa*K`GAl1C2Zen_k&1Sf(ZA=uHE0T8LiR!)ADN5%bi9?JZFL3lpP~To+6h9$7A8C7y*8YZi`T`@?iDd#21>51ggc_;f-bGGhkn4RjJL|& z1$Q=RzF)D=@=m^W-}c3%0EwE2>~6d9tkU+@dxd6UzSNj@w522{8-KJFUZJh^WYIA@fL`)r7Bnz z1_5>=&0h9N+Fk{DW5w@yclk3}dz7j#+QU-GK)Wi0oL(6$^+j;E;2w;Yvh4-M$8z~d z3iNQ!?u=Gsg4Ex0*&dcoy_C8V_g9ck$X4|Jm>ZHC(rQTPgWOv&1=Is2c-zB=ByC@h zfONEak1hJ3mr=L5K@+h2Y%QFB42cZIDCKJ@&CDQk!XIurs~;Uv#^Ya>Q(2T>14PD# zygF&*Tf&A_h(V5&peKpOEQhLXC2XHzy`Ej6IH7bs_G--l9xt?hQ=NZ&0VO}OrBYcqx|Oo)A2w9(recIo`n7_rcM10?fmmR!h(Q4 zp<1-rjLg|DCNn8q2hky?b4QQ^Lr*G(YQFv1Kl808z~IWkr?=8^Y0Pad6U0a0n`Z)D zW60MrM(E$+t_t4R!jSLc(}%t0wd9V-HvwG?R)7bxn4kgcR%ILW`(C;v8+Nk1Kghk~ z{ImVa<$h-6?Z@u7-9?B0T9Ens7`AZu;jl4M1_t+dy4f=NSdR9zIAwas(p~O^jKpT6 zc~euz4h06zKa@PtCL?9``@MN9>5n*66W?TM*NG*ZPFm&(QUkKQuft zPPe%Ir^X6wsLMToHAkk;>TqNk_OS}s3GC>I+y7ne@VFFDgks%;D%!qJNmzFbp8J?1SjZ^tFd?lkuEgnkJDGW_o>3?BOqg z>l1%nSF>5jvReL>Of=|!981|~XP?3x2pj)kkTVXRf{A>58@!9Tu^Y_LT)aVQ*y!-d zSnzL<{1k7ju^6;IN_1Z!Vsh?^{{#WMeceBK=9S(X>_5~9u(_Q4;%T^YS?!m45og^W zR$QiIQ!~0uv_~qM)?8T{@u>9}BSESGj-`iD$7*}UJYy;Ki}*HTd1jZ^2#EX54bsh0roGK;7O0cLcjZ8a;t=E!cblWQ6c$iZMd4Pzo@hc#F@+X{ahAX zDj=c+edHn9+)hyX9JGtf@caHM5 zpL*Ue{pVPenrfYefkp)*cs&SkL~l@Zt%YcTc8Fx=LuoCm^t=Y@ksq zz!8n=dJv*o7ZD9M0*m-RaY%#2Um#)%Le6$UJJkZcqN5^oZOxy8(ud2evq#R1MxEQ( zIrJtbCf{!DM@I`XhiVsG5`r3#GuY?;#_K>L!*43$aYE;F@bHHQ#hsqMSiYL#%kOmx zjIpdsf(Rb^MMV1;{v@(8s>bCLKFfN(*N+Fu`D!*t4UWW4FFvEkH3APz4Y7Ar_tgOD zE5}rz-cVd{*+7E;q>+!9L0_$XcU;)(6q8ja0m)nqtQ3CG%3%HSNG*6%eTDBw2(dgM zs0)ErW!{xgQG`qY``jPBnYwxK`$uBfte4AU(EY8d!_hjKoRZ5b5VgR)VfCW!)!O?u zXiIBkvHM6iwi-AfFAoQxyV@LvmE>)MibnGKb)tS9Vg%!Fh8Zj(ez?c_%X!(|~X zYqEA5LSEYx5u->GSOi|W+T}WQP99qT#N9E-F7R=e5JPo+y$iBzLfb`P=cSaD#e<>X zw)wB6NZ#BhVp>kHm$D^?>6g5Xu>50GbQ66)bq_m3$F2X!L}5&=pCcOEwWF7_ID>&9 z@+pnDlUVz2$UO!J2XtB>g{!p}Z1h5ZL^m31K{hat#mmayj5=g&$f*=EkJ4P9l*EGo zglBOEruZJGKl)do(K|6C@s-+lxbWL2-e8~>Gm>fFrJh4{%J|{!*?l2xkz7abfTtYM zo7H_$Q?CW`cGjtUxjSQ@KRkYqZQ~oWPv+AD+Y6{?dJl5njy(1M)|QfIY7^H*ta)n| zpK&s0mFw7~WrNqF_LvQGuHXPvFSp2)|L#wmgM*c9JGQyHpVdH-P%EwHuQt&?v5l$~ zI%>%>0c*ehFHW#9QNh#+Js`5J0qo^a9xvk47i~*`^cU7-ittIkcZ!sfkRuV=kcRDF z%7T*`%eAEq$YXOwaieoJNT}{XP2J4s?x0xBb5AOPw41*2Umh3ah4ja35D;;nnE@>4 z&I~7XH^m$ZAB`57k&1fHZADVTMZ1;`la1K&Q$NKTIpxtdW~TFc0>z+PqTRr2`)F)h z1z-}3&kd86I);NAru)n3#K>_Kv!Q1!&&5_C$aYt?+8CI|?+QP|Qa~p( ztM=3=5IL&oD(}kgqwn>Fs~|EOYu>BsGo`vl2zUnuLBBMnV{)bek3KY!FL2=x9_k># zuT3F?$F8grZfeQKEvY-zaZQW`yW==M;$FNft1J7~{Ekr>%Q8U{9GCg|_pR>(=HD^S zKEdGCZj=*@kbaTKSN0Lr2=<osK1W4zbCmN>{F4LHTIRI_ufY;-A*)t$yc`q%_y74vd= znJwn$;mhL#qf{m0REbDJ1u`++vNlY>q>7PW87n^ZH?xH2eKWXiug_!G*^j>|G3i3L z@@SoFziSllH(H2tT;j+JN!fh^twk~}1}g+t9(%!7=Ptx0o2gY6;aTir{eboy<(_Z) zV%y8X`j05Fz?H5Y%fc!-Q~R9YL@dFrycZ>tyjnzi^w!BKw)4QBBD{IWlPaLUh*YoC z>pu$I$^1QUF6C?BHX_+0krtSTzupX8a)l{1{4v9T)lF)TD7AEx5cDUXEIN>+zAQ=} zS8d`E^+HSW`hM2>7BAv~B{RPS$zq8jI+&Y*00;T z=|E3OaPY+>EUx@hX&%p&T2ME!O~DO5rINe<271KSGnm0LfD~{PkVOlb>(y~X?=HgY ztQ~WM!)EU~xHCgc`3Ml@BUJ+&b506!vLFKfe$ow1O2hl!{IbT z=hvv3Yk4mPfDAl{El}18c&WGiHdn42oIDkj3eM-p7Mcez2q{Pvb)&Q+Et)}*$>_D- z^50zrZm_JY4s_^BVGEFl--8uOrt6U0sDk}Mr`&#e*<%YIZ`aMkmyNE9Ot9IZZ?tZ} zSk?k?oL>H^8_^;>@xb~pZnpGOMi1d6WbXPc1?Z6t6)KXO78_-x#wvx~s@Vz%hH;t>!5 z)?p2Mh3@LL$U4FCyC~#HOL~uv5T_!pmTtJi6|=D{r(IArBX!9J{Uws1g^TVwaI7fA zaZjEb`t;&`R|QY$d!Lf?(#Uke8zg=$fMQqNs26%pxVovYuN~+$p@8L&klPL5hiqE# zC%j`RPi!=_D*L#hhjr)|TLY+6;HnEP=COI9;|eIChNI^~5zX`$dS@S#qq}^GLji$d ziomeqk-GtNdBU&Xf*Rj9Eldbk8@sWNKYIR&c3}}6k2rx|8v+h@v$Pu!9N|aH);6Qa zRr;l|hj0#h>IyrCf_&M$uEr;jlv(8wS||$X0G@fGlY)@G2VZlQ&A8Yjba2pSh+~g* zEfT|-$@1RvHc;!b9gK7hBBbL9qh0wbf!nP>SaWsNPz6qt@KzC#6-h}P9NdcvELcCfX-3U`+B;;-|cD7Hww2SQIxg9HX z1Ks~X1)=!jDmrQeXu6JvhHsyG-~Fr}h;K?ph)5tR=(t8p;tKzC?CK$07}w0K1HMH+ zd<$O6i_SdffHoMf#bGQe=Y#3%YYc${wf30zlOo)?L!2W0WH8Ixj~g8OIWYB=5# z-6`N8ouVS#(9A_daW7w6m%DfIT`z08b!BYlUUIZFPN#ITpd`+hn92ayL<0_JQovof zjfjN`xCHC8*{Bu9o>w5p@{S%qx1lzv{oK9|+;*EjZTkW#y1X2ozDu>%@JOZD_v}2p z&Aw3A>t)iP^{_e*&IB31Uhk#IK%u+AQCK?0!E! zik^cGxjb+;;JICJg&6Gb`0AcS90|4z0a^Sb_tF?%gS~_`Jo8&6?~c62URL{l9=Te_ zFl@A;{FfiWSpAM!TcudgxH)lwP&M!BZcKnVKk#0Up7;hbrw{zm2+T`@I5WSdG$AZE zp#EBNC_&0iw{x>z0#rqY#3%e|B%H{Piw0WXjYv&-E-F9-U|^oVtGzM_sq~OvSM9M* zhSZ0D-XSS|#t9`7vp=IdcnhCt2>51wj@mJ^GnXwT=#HSI)##5Eq%;XjKE=dYlxUTe zcXT~)y-~Rkepr`fgZwlBDdMq{VH<70DNoo~#6SV^W&|$uoT>73n{B)E%r#j8LYXC6 zsTI5h+mu)1*JXTdvF0s6^%%gt9pP~d_c;BjG>)@xZ&CMOhA=WKe_No`0yLw_Yc5I4 znMj_s0sAhkZ1=rSL|NRMn@w)>g7GkT^`w{ny&Qb1C$RRLO>(G+X5$gHKsYZBl|1Ok zzAceUD>4<<6OKtD#C-({{f!Vde7#_b>Tpy}h%$~0Wi`({7z6NDb3gLv7|R+%-W>H9 zY^@$_M9p2(8XY=f&U7##!FY8v~f1xlZ^(<68>#Ayxs^yBlRbb90s7 zm7sx}wG3Mkg1oiX0z9gM7Z%<1StN<{8q;E8*mcDtHL^xfKZ8O5q)R8hzS1b zEe|kz-RuJA|5E(&7S#D|hOC?kWunRk^^cD}aQuH-y7EA%zPC@4OL9dhy4J`NqA+96 z2uTu(lx^&)$xxD=F&g&A4Kt}Kl;qP(~7@4bJY`<&-I=Q-y& z=eeKHbFIWrR^IcX@O+1(`Rwgq^#dhHOE>-dAn)CNrh1kuf_W`9R$^H#5q8sFRuJ{_ zmj054n-4RL+?t2%bJHJ^+?cTuLo(Spqa$LhjK3#8u|(Rbv~+>`T>zE{M|Jc2F>ph* z4-WYfwkr`u&DD=P$?SZ0L*vs5E|ChLK&3bbZs+DV&wyF3s!tDPQ@dKF&s{L60OKPBa*P^{9PTlvGp$4PVpD<7tiD9>XoDgS&WT0_DW z>j_66COM8{76_l-N|i%tUadt5P6Ofdb;t*x|DwDD8NcnO^)(dx-3W5P>M|pd{{$YH za?BjYxXq4QWk(({V7_yez=^C0P60a)K6@bcN9IxH17q>?Lt zEq-w7+w)#szVBE)f) zutYMiu)r*kM9w{P#fOxKzI0?1m}g{wwpI8a@)xiAz$5V{oVtxjvFXpyMA3)pv@)gv z@dv9X$)LrvA>s~!|J|`6w)Z@f=M!~DiK)Y|$+M6sOE}xu0UkmT6+j=vBQoug&YcTT zNiqn(8yZi>Kvmk+bNK#$ki+2V1OtNPD^-1VZxNgfd5h-i+Z8 zZu`_NHt{srAKD8mQdz>#*QuHYB-dPS5j76f1$gCqwnwXvSws(E{@t5}YKe8jOxZ%vVf<}=H^acu zhuL5x@tJ|S=}U*NAz$6BVSE2KaD`g@YPQ6uX+SOn*_Q=rdQmTFLkr*#bwtfr!q->n zD*)4A95Sa&eB`gWKl*5YV~SWUdVzEB*m;_U@`W!df%fGk&aBV|4FT~Gjgfsb94oR1 zcOsW2r11oVOuSPXIlljU4L-QuViweHe})ex{HfIccb(|Cvh3LkRK1>C2mNJDshHNX1%r12& z@Gb{_2~Q0FPntYGu@{cr*TLYSBYER(U@cNR_Lcu5_(pSLk*ZOCut@GFY)2nNp z0PCMrxN`wqszc8R45)WL2Jrz`f%mk62dp|^?IVeC+Pp5|F3P&^!H?*9DsJLs`%W=wyFjnqIV?=f!VkXnj zag=RO7@SKi({uJfpk1H35vpFSbSsp;a6?9pRnbfMoc~!jna#H`h|pkx#H<1Pg`mW@ z6?kTBL3!i00nN06jMNw(WjL4;@4 z|5W9gBamqh$`IFRr@_XtR^HXhvxC^kf6CIvYw&2MEDYjz3DJNt78`lO_3Q%he0+sD zUP1=yb3=$Mz{Jg2EHfps@r#lt3bok!uM)yY=K&tS|6KY;}^$ zg&+}FN9jZO)^5_spBl1r_y=<3K1k^@ztQ+#XB+ZTTh7kP?3?)jmPQ8*b`d^Vtq`~j zyo{z9k3jB=7)pI;(C(FpUrmF!g3FA^PY{bllUi-%Egz;|%>_+Ni;nx|tsV|IF;E{m z=#&oUa_H2t8oaTg=Nb_9_xRo*0#Df1_ig)oKmTMcs9Qfqv2jW9LGEh_e~<9|z2~;N zh2hY9LIL->v0heRd4KJd@`jB$KW_quQ}m@qSzvrzs_m7k3F zNMy`b3baC4bEywpDg)c)YCHQ1*MLY*{umJHL)bo{JGQujG1kje% zEFjCz!5+Fp#h*sn#nUmfY2aubHTPQXyWSPG(IfH4SAmyIZK^Wq{mS8}x=?_u&i8qkoeBLMN-M;nbmcAlxHm1cO{ z5N0?w??W(5kC63OYkxqCKdCWvz<>a6wSVv}by=^Sm1P4=aDAJqMISO7qvJvIVy<`; z9V*7_#=~f3?SsghU@y^}&$~>x5;iD!`M-Y?8}>4M1xDGd{3l<8OJXG_2I8Z@PvyVG z{6_~*rKr3>c?Syj!=b5$j$yR}0_E?z_3*F8a736Rn7b~FO#RX;iaY&-U(2z1{>&3B z@p6pD_C~QV%DW$O|0OuEBUxsX6OFQTBelV^x{tkRZ95C*ABq! z-Mk|SpEZeY1r~!4=eR}Y{@F(I{PpJRJn%P(bxr_j`V%nE{;Y5F5is+x8~pKa&DK;c zfGg54Ql869Eqd^>*zG2HH>=tg7z940z>Z;1)mz}gBobuq(sC$S!J+5s(Uq|4wt}{& zkixnZw7q6ha{B+4s3=x4+Y5qU_!9WxAc@9)yFaUl{Ywm3^^uf911ys9iJTnc&?eKu z8RM4hG58dO=H9d)`p;fj|Jf_eu#Ds9c{mhy!DTJ6s#%Wr<&kzW%Trdupv+HgaW+aB z;qHTd(A*;z2?7lEB;LmPR=E;{d|eWrr+%c+tc9SG!0*jY)NfeThd{=#9|4MreI7-A zI0&h0`odcEP}mLhW@|lDyX7PLgv@7w|rTVc=(6jq+ud zKO?O~&(oygbcTX^K_`$gD*5Q0tTB?Fb>W37G8L509BGOBJsk8yVP5yk$mjc@U?(Z1 zJDwnn@|^)^cIKWef)5!?>r*)c(_oz(J0vr!AZGB>Wp_u3Tj9k+tmNY)gV*Q8UVr;W z!cQb%eg2!FicGgPz!R&={;{)2;pqvS0SyRaV6_|bk26$V`=3Ycbpu9b&lU{E*W;V; zgs~%^fh&z3_E)((Hh|a)g{irINHVY!zS^AV+T(`kw@-ht^XCFx@iF%<$%6~ z4T~@^lIBvSMX0$_Q>nW_1hOo$@A|^si=t3_4F7(lkR{h8&T_R7LTe5;ym1RiL#oeH zu}Hg{7ymV$`t4`wPIxwVd>FwnKqlIiy_5_-|Ct*@sG#&LfJZ`ZRh&j@o~)q3@x&rw zR8oWG1iMcig;pzzw0khqA@t-9+=CI18XKLcRDLOh@H~h6qz|(P%Il`HF~R5mSvb;; zoq_f3iKsc1ES(D8Z;ETl;Z{4o`IR>RWt6xdS+Z4Q4R;`X9x%pPR#S96E7=QCb5S5E z`&DzN0_~w7^9>U8kZsPD+B(=8XSc@|OT6Pq!fgof zN@s3aR$`A&+;8vErs%^8t|VYzykuU4%=56StY@jA$ZXT{{mkyv`=gRUt!r>e597!C zwhLca2wzK37cx?gmV0_zrE`9GlvoJsN5?ST5siYk2iE}O&Q5W*hw!QM>Ir8-hNMRG z>5sA>h2CFIs8U2#35rv&z9d(~v=GDmpWo1eU+@>8A~Vp8QKCGQxP_(8f}f^WjF_N* zS!a|7!AjUcHbgyjU9XHne)Y~3aVzTPw0&Z~wJ+T4QhUoAlqMqokT$$uAG3WoifOID z`!k7DMczZ0HN}s>10l$}ygh;gx*VK$?H8aYllo;5WXp=|g%WMZ_>ZQnzwsV~WW73> z(=q2hlzWS**^}`BA_J!XDHdA5{{%&Sn)<4qy}9ADp6~bWWtt+ihC!w_{AmwAVpY?9 zaAog^`j70T>n*^LVnx~&>EG~jvqcJTTDNzUq6|s|^l>a6q;Nl0QcSZ3d^&#YyNlA* zHqkD+K|RmS;+K5N9QQ_vP5*kJ?ZF`6L5Mrxbjo2U5QCW2n$>}<<2t?N+v3nrVb`_< z4tXCpI!`M=Y1&n=zeHuT7v}dsPhXHZ5s?vHe+8*YWfvhzoeI?u7_6SKfXohr8dJ z^`B*>Pk*=_Oc195*ii#jQuz8uKw3Xq!Da;K^XWE`PGydIns*!UTd`t4GA`3FY?R39sHBJ^ z`Z-D#dtgT~_t{d!AbBJ24qxgk7OJ}izt(nvX!k7gbfp+1oZ<2}_kWrlvXnWw3IrlG zF+~cGQJRDm*A{G_A_*VLq_P%g_iOGvazpH!W`!VjFGPDQY_0+i2TR@vj zK?J?yL08GNO6)5Kjr>DyS_DLmw^cXR4?nn$*^U3b@b$EZuUe$6 z(~yZM$-RSDAAvbHMEr{z0cE`|;aliyn?XqvHZFb&>=g9Ipv3fN7=ZO~rW>MitGeP? z4vM^vk#+AJf)Ly+(NGe*=$ErXNTU9J-%}xerx@PmJ`9XU2XIL7RFIWa0Wxs83y@uQ zM@TEE;ra%Tq*{(L_XybYV@U)rujp(1<73}JPUJ-7CP(W-KDB|mhW>}3d9_RJ1MV$4 z5iT0MWX4^tPhr>Q?a$v%m;ep*1T(RbG5SNux;zkRAq`%IL|w+&|73fHRJbG3g&x@a6L;ms_m)Ok21Qt-U zW+zXjqP+j6Wq~W#3EIavu8d-0_v1WA*(eWD^7(61pmz|t--9r#-S^Jg4bgfj@%KdJ z^5))D6p6PBA5I2b?`mGUHzx!2wZ z71PwDhvelXe|w3WnY}92Ci{DJ0wyUBUWDOp>)%pEH82cz&Aqk%5EnpOYWDNl*fW8M zA>;Z$7liy;QUaDV-JE;#tCZK80S`6)^4$x-GH~75FZCis(e@(rbUzk+#z#ik*@)DL zC4!P#-y$SBY1<#d2a* ztpc$*CpDr_n|nu250Tjq{l{LKku@82MhxdKL|@nr=Zt2lscXqhe!$B3801jvr2`FpQ+v&*DCRkP^P~V z2%$rm6`1Rj@ad1bHC!UBTLVx7$8{_Uyb%w{dle2*eJ8o@M2CLn14&a{AOBZt#R~cU4yfs zO}Y;n3qFt>(|mr z>FJLG>97EDDJ4T8%9A6J@5VuyR8;#wQVlL!W#8Qp(Hu4`u~p?YxEcv(*#u(Q5bdwi zgEoNIM*VY@v_l1zES>Aeq1U=b1({Hm+)nr2uKA72(V$X2`h=ga?9%r=qG$FZeWL+2 z`9TXJar7~GMifUiBownluTxje6&)>*K?`GThw0Wc1Hlh)t3&FqlJC-~RohCBG&db< z4(^Dx$?k#tlTVq_oX#$R1KEOauB+&UAV#STM@Tf2lg3RByYA~}sR42u3Zm*5+y|^-pN37N5v0AsX5o& zL)*$e*R2D?y8R*~_H)`FyVioeT7e?**JH1WXo)GdY>$6AJOHuv=%app9YQ&k#qj<1 z$Qxk6_Y)qbopEqpjS0IOJ}oGmz44c~BD(x|QtXOEF zgiD?Ma~8)HJ#!xNKS^hc$SGjxJc%;rVFC6db|-L1)(=oTAmQpBD)2N>?{bGq;+D*% zmyH#M+o7jw2lkgb4$fRo0u$@eIlm+p*-e+`18=#K*ndZq`=gGq;e6R!g_6w3XK=o6 zFV%`lLaFLW2BM+bS1a%=nos{{Zn)&fB~PZ=d@P@L^FqS1-<2WL_my4vA62uaMoI@E%dtEpy_o&xsgBOqiFJ*HmK)Dw71;;R4Z3UMQqiJ$h3jvPA>Tvx z8My!p_!1^OnI2$xGm-`5C-=mFC4Fu3htKs#4o*u-tZXTg>!%p*eJ^!z=OP~!p>_Yg zzc7B@r5|8t5aUYuMz4_?}oS7|vJyp{33+-!h5Z6;5 zF0XA35jd@WnA7XK?$s(ZH&T;HC&so)-3YprkG>s5IIM`Qk(4u^pblkRVUEN_JbjLf z{_g)LZc}F>`r*8+5$3}~mE1s>x2H*0)T}yC4#NDgl>O-fnqM9QYZkwpWR7|4h5Gid z3Tk@)T(~4#0;L&nUP;N`EjkiWm-GYs@Mv%Z0n!0w!W2b^OlKQRzBF_LH$WdX}x)h_)p5l@otoa zBh>6hIOUhZ0`fvG9f{L-d+sBPW8G*4@@lM1!xj-Uo-8_>FdweL@$Z(igOlC}6L>N_ z(_tz7>_Q_^L!=*SKE&x|Y8==Pj5COw@L+x}9{;h{nsFmNh|rM?YNj{HH#sbt<61Aa z$xTHL506GH{yn^ZDb0a8B<|=NQ#RECD&M$ZvIdY_XiksQPnRI$+5ki zs1uiUYR@cqL!TtO zS$ga#DsD%sH7v)rE=r+_M01uTFVUD8~S1QIDpA^aI-?p?^y8VN{>e zS}dtYbA~+NZjR&obtlnlvBzP%*q(0CUu{6aGiSQq{W=Y@rvFU#DAmNQ0#BqMt{Z8q zi@kxqTdcSS=yR-HZ=SHcs&qt!to=?-;&*T&fe@A-yp=Lpca~$QjV+-!C!-`wt*(#=(_&yAk}H)G%fu>9C-9rrri(!V-D$KRfF!9aJpYv z_6Wrz&3pGT+ejmjGro5cl*HTKK@_w>YtON4-!J6|;W%@_qi?+`U76axaVra8QMGiy z^*Wr)qv~kzBcyq96b1w)MJdO%W`M(T!unA2^E3#ypXGlI4Fu=L*~{UFo+aviNW!`5hI_k5SD-cH5z={z%dODSiP{8 z$Mt=y#INp^b|r94V9Y{-((2K#1DeD7j~R2{Z}W6P+*;*X1$w+}d2t2|g~ER_)AZV_ zy2bW}wY}~em9*BGs9i`n`c|5EKg8nw_*qOI#~jw1WdY?vbm~@f78$=wyXc^{ZxNn2 z>81TD*Ud$^4kMYZ0@*0mg(q4Rnk4}|aj+7{3u5QtrFEd}0k;U8nT-b>IC)M&fffZT;HMLwMDB6W z^|Fw6ym})li((_Vhm3v!r{S)5+_93vqaSp)gYTGMq&-*E*CvUeB(@ca%Z(d&@;mVg&_dUuo3| z&=;ZLf4*vj_TSUxG)fVQu1X0%T90?bi;O~Rw~Ud#4)^mul`Tqz41Z*T4iQ(zyWSt| z)jJNG7SE_Z6O4d$og0l5=S;&g>E>tX)XAA6>mLI@+&mSHVh!IWc{!zxWfJCL*Lokh zXB`Otl?B_}ISbtJIi~vNoSSP<-G1TMdx8)jlsU&fPuID@GqydKK_?!7-0lP-OoZ=O z;Kvv#E{KS9w!!4Zfi0NT?PSl}v!i%zJ275A7mOmYpQ$yd$B*^AV2)y5b;I=bd$Li& z?A31X9n;TI@SigYce1>-h(TMEWZLUPTEFM+DO97FHR?)GjmGDa!12yN^gUB252r&o zX5}O)w8?x5W`t`VuATZhg~g1XhBp%=53lDiv$ zYSe)RaN$?hop2oW##PQr{2=nPf*YV08rS!z4cfe^r8(<5oj&PF;KQoUk!gD^&QT}N z2ji&rLKr0JFz10*orROPgw@;f=MT()IWs>y`l!yiAN!QB7>8tZ2;$~wp8Yt-<}ZM< z7VC+axAUXRkBW{7u#evHsF&>w)Ja3J25v`2o_EG+ZN9lmHy}w0F(UciB>u`uhj+yg z#B+>EC4RjjnB<8-*+i?b=pIU|!SH=9mxEXRH=o{yv=eGSl&y2P3Z>ffwoSGkHJ3-i zztPlGpTRO^gm0`lJnmVp-HIHgUOGMT%U|K=KgB*UB0~rfZ|^Pj#7o1)ibQb9+}o~v zf@IZ(dO^fnvRFyp67TVq{hPbJDMNav**bFm)`gg;F-+$B*Cq#@fALgYyL^s354fFb z1S%S7mUz&7v*wJtt#LF#Xx$1$QcKxXftKtPa(}qEB9@c)7@|_EO2k^qpoMlG$dsE? z;1dS1|0nH@VTzN+x;07_!5Le#Grxx(14IxM7LA6n>huQ_$yGFkri2-QvZFcokumeIIQ##iJ{ ze))`_OjeWue@g!IhqfY7q^!YgoAub^Woe2!0sSbZdXrBv0#Ms*~7)u;-HEu;e zj6Dp{KZuAud5@9az=MIvDC7?}EVpSUs#ziMLj|zY8oH`0W4l6%-t2 zidP>~*5uj?zOKOkOt(TNSpi+-)t8eto@q4OVx=wHB{OB@Lx)iHYCRSaR_tZ(w`F`JTenPA?CyA&^HYzQlP zYMhyT5SlX)%=TfkQ60nTnvFyjCfp~pU-m19>By$`(O(`cA2tk`d*>s0fRmH*$KjYd zS3IbB^dsA0Jl^@W3=&Dy?1%sMK&5OA6xlEgY_(FipJ+)AzJbY)dDH&O7u~%jc{qL0 z@nDrSw*`uACRJy`4MC`*B~D`euxuI6&sQ`84}B#!1*$Y}7Nn{}3m0Y|%(wR$tX&aV z+@(Ccg=EId8;r0aZ?k;DHS|X!y>t(^1Ehfd`wJwzsA$|@ZM}r~0GS>j`Jkw7xmj1h z%*kM*`&d;m?$(5pA|IuGp4}||3}*W*Nix{fAFCfbkCo)$RAU$fD)4uw`>?7SI(Wo% z+ZRpSrJv&q@ft{a+yK9$Gz-+{F@6qdmWhbULk1L=v(s^^*!1WAgVbT#hH$=}h#B0d+aCX7`k2rtCfH}J-Vo7*;OTTO%yxg7!c_+E4 zwN(hcB+>NmSK&pqHFdog+cBYbDMN~BCRTBV7 z&db2L96?f+SlFw>ShfX4(W)-afh%0(0!$nzs&=*TE@HsI|8-A-F^ZX)HIAh|)<~(J z@VF29-u)z->}?w`y{Nw?==7UArEu)x=G0cUyYTRzz4lnD(+wULe>TXQY(^Tb!|y}- zE`;N~5c-;$L++x&E{LLU*}wO7jW28>55gaZtZa;m*t?Dy3W13vTH@|%yb)951i^>~ z@DfkFQ+)sDue0nAorZ08aUa2L?jIVTz=yuq!paN6ToB>knJi*`F-;p8%d#7DlOa^U ze5=kEXCcx%Ea3BnK}8Q6MuCBA8!w-GR1eFohH>oIIM|kXL!v-5B*E#f0fTS_{!PJ@ z)qujt)tVx-eTH$5NaTXciBv<))+P~A(^HY`iZ|KimR^TPEkFJ4OD zUV`!x=36Ms`G?5!6ZLh0cO%n$ZLgiXa_6$q2Uc^PEk=h|iNI1nwDxV^a1w_dL-Ack zmDd&Qp?2Iog{f$#mDSqt9jY3q=MCyRJooim6RQc*2Ae!$d8I^^Xr!d*KJeuWC(4~3 zxo?|adfvJ2Qz_p3j6ZfNFzxd|Jdf#`-0_Gjk6I3Oawyx3Kdy*j@LA+a#GT%y4}5hR z$7E^kU~)$?zbo(doaqEF-W@i*Wl%=OT+mxwO23z{*SxCFnDF#V)vl5dvy(ArY3?&@<(&K8W! zg1ngeZ}*w`^b4)f4U6vA z6XMHMchvfAE8&a=Pt)$; z;)s-p)iv?xK_z;oYs7cr+=T#hw_SCbzOuIit-cu9S Date: Mon, 28 Oct 2019 21:11:26 +0100 Subject: [PATCH 027/257] Fix downsampling option in querier URL (#1562) * Fix downsampling option in querier URL Signed-off-by: Olivier Biesmans * Fix downsampling option in querier URL Signed-off-by: Olivier Biesmans --- pkg/query/api/v1.go | 7 +- pkg/ui/bindata.go | 142 +++++++++++----------- pkg/ui/static/js/graph.js | 13 +- pkg/ui/static/js/graph_template.handlebar | 4 +- 4 files changed, 87 insertions(+), 79 deletions(-) diff --git a/pkg/query/api/v1.go b/pkg/query/api/v1.go index 0152d18126..a3b97513ce 100644 --- a/pkg/query/api/v1.go +++ b/pkg/query/api/v1.go @@ -207,11 +207,10 @@ func (api *API) parseDownsamplingParamMillis(r *http.Request, defaultVal time.Du const maxSourceResolutionParam = "max_source_resolution" maxSourceResolution := 0 * time.Second - if api.enableAutodownsampling { + val := r.FormValue(maxSourceResolutionParam) + if api.enableAutodownsampling || (val == "auto") { maxSourceResolution = defaultVal - } - - if val := r.FormValue(maxSourceResolutionParam); val != "" { + } else if val != "" { var err error maxSourceResolution, err = parseDuration(val) if err != nil { diff --git a/pkg/ui/bindata.go b/pkg/ui/bindata.go index 3ba481000f..1e44bf2508 100644 --- a/pkg/ui/bindata.go +++ b/pkg/ui/bindata.go @@ -160,7 +160,7 @@ func pkgUiTemplates_baseHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/_base.html", size: 1478, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/_base.html", size: 1478, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -180,7 +180,7 @@ func pkgUiTemplatesAlertsHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 2696, mode: os.FileMode(420), modTime: time.Unix(1567492419, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 2696, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -200,7 +200,7 @@ func pkgUiTemplatesBucketHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket.html", size: 1281, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket.html", size: 1281, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -220,7 +220,7 @@ func pkgUiTemplatesBucket_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 785, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 785, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -240,7 +240,7 @@ func pkgUiTemplatesGraphHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/graph.html", size: 2298, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/graph.html", size: 2298, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -260,7 +260,7 @@ func pkgUiTemplatesQuery_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1364, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1364, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -280,7 +280,7 @@ func pkgUiTemplatesRule_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 961, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 961, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -300,7 +300,7 @@ func pkgUiTemplatesRulesHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rules.html", size: 1944, mode: os.FileMode(420), modTime: time.Unix(1567492426, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rules.html", size: 1944, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -320,7 +320,7 @@ func pkgUiTemplatesStatusHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/status.html", size: 1272, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/status.html", size: 1272, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -340,7 +340,7 @@ func pkgUiTemplatesStoresHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/stores.html", size: 2174, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/stores.html", size: 2174, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -360,7 +360,7 @@ func pkgUiStaticCssAlertsCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/alerts.css", size: 401, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/alerts.css", size: 401, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -380,7 +380,7 @@ func pkgUiStaticCssGraphCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/graph.css", size: 3844, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/graph.css", size: 3844, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -400,7 +400,7 @@ func pkgUiStaticCssPrometheusCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/prometheus.css", size: 470, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/prometheus.css", size: 470, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -420,7 +420,7 @@ func pkgUiStaticCssRulesCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/rules.css", size: 195, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/rules.css", size: 195, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -440,7 +440,7 @@ func pkgUiStaticImgAjaxLoaderGif() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(420), modTime: time.Unix(1563780416, 0)} + info := bindataFileInfo{name: "pkg/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -460,7 +460,7 @@ func pkgUiStaticImgFaviconIco() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/img/favicon.ico", size: 15886, mode: os.FileMode(420), modTime: time.Unix(1563780416, 0)} + info := bindataFileInfo{name: "pkg/ui/static/img/favicon.ico", size: 15886, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -480,7 +480,7 @@ func pkgUiStaticJsAlertsJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 1152, mode: os.FileMode(420), modTime: time.Unix(1563780416, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 1152, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -500,12 +500,12 @@ func pkgUiStaticJsBucketJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/bucket.js", size: 2834, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/bucket.js", size: 2834, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _pkgUiStaticJsGraphJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\xbd\xeb\x76\xdb\x38\xb2\x28\xfc\xdf\x4f\x81\xb0\xb3\x22\x2a\x96\x28\x3b\x3d\xdd\xdf\xb4\x64\xb9\xbf\x74\xec\x4c\xbc\x77\x2e\x1e\xdb\xe9\xcb\x76\x7b\x7b\x41\x24\x24\x31\xa6\x48\x0e\x00\xd9\x52\x27\x7a\xac\xf3\x02\xe7\xc9\xce\x42\xe1\x42\x80\x84\x2e\x4e\xcf\xcc\x3a\x97\xfc\x90\x43\x5c\x0a\x85\x42\x55\xa1\x50\x28\x16\xef\x31\x45\xe7\xb4\x98\x11\x3e\x25\x73\x86\x86\xf6\xc3\x97\x2f\xe8\xf3\x6a\xb0\x27\x9a\x4c\x28\x2e\xa7\x57\x64\x56\x66\x98\x93\xc1\x1e\x94\x9d\xbd\x3f\xff\x78\x75\x7b\x72\xfa\xd3\x87\x8f\xef\x5f\x9d\xde\xfe\xf2\xf2\xec\x0a\x0d\xd1\x77\x07\x07\x03\xd4\xeb\xa1\x19\x83\x46\x97\xa7\xaf\x3e\xbc\x3f\x41\x43\x74\x78\x70\x70\x30\xd8\xdb\xab\xc0\x47\x7f\x13\x30\xd1\x10\x8d\xe7\x79\xcc\xd3\x22\x0f\x49\x46\x66\x24\xe7\x1d\x54\x94\xe2\x99\x75\xd0\x14\xe7\x49\x46\x5e\x4d\x71\x3e\x21\xfa\xe9\x82\xcc\x8a\x7b\xd2\x46\x9f\xf7\x10\xe2\xd3\x94\x45\x24\x43\x43\xa4\xfa\x0e\x74\x21\x20\xfc\xe6\xea\xdd\x5b\x34\x44\xf9\x3c\xcb\x4c\x85\x82\x8d\x86\x7a\x14\x53\x63\x0f\x86\x86\xce\xd8\xb5\x36\x12\x05\x1b\x75\x89\x0e\x72\x50\x0c\x45\x8f\xb6\xe8\xba\x32\xfd\x69\x1a\xdf\xb1\x29\x7e\xd0\x73\x77\x50\x4b\x30\xc7\x68\x88\xae\x6f\x06\x7b\xba\x28\xcd\x53\x9e\xe2\x2c\xfd\x83\x84\xed\xc1\xde\xca\x43\xc0\x88\xa7\x33\xf2\x1a\xc7\xbc\xa0\x62\x52\x02\x8d\x60\x19\xf4\xd1\xf7\x07\xe8\xb9\xfc\x79\xf1\x17\xf4\x1c\x7d\xfb\xfd\x77\x1d\x51\xf5\xd0\xac\xfa\xff\xa0\x22\xa9\x55\x40\xe1\xb4\x2a\x84\xe7\x19\x3c\xc3\x7f\x59\xd0\x47\x87\x7e\x8c\x18\x27\xe5\xcf\x38\x9b\x13\x81\xd0\xb5\x68\x7c\xc8\x82\x0e\x0a\x0e\x0f\xe4\x9f\x99\xf8\xfd\x0e\x7e\x0f\xe5\x9f\x6f\x0f\xe4\xd3\x54\xfc\xbe\x80\xdf\xef\xe1\xf7\x50\x3e\x1c\x26\x50\x91\x04\x30\xf4\xe1\x03\x3c\xc1\xef\x5f\xe0\xf7\xaf\xf0\x7b\xb8\x84\xf2\x65\xb0\x77\xe3\x43\x2b\x9f\xcf\xe0\x3f\x02\x2b\x1f\x2b\x46\x25\x2d\x78\xc1\x97\x25\xb1\xc8\xde\x5c\x64\xc1\xd5\x8c\x64\x63\x34\x84\x25\x12\xab\x27\x1e\xa3\x34\x71\xa4\xa7\x3e\xe8\xfe\x3e\xac\x6a\xaf\x87\x2e\x09\x47\x09\x19\xe3\x79\xc6\x35\x0f\x46\x1a\x88\x7e\x06\x60\x0a\xec\xa0\x5e\x49\x05\x4b\xde\xa6\x79\x39\xe7\xba\x95\xaf\xea\xcb\x17\xa0\xa8\xe8\x9e\x8e\x51\xe8\xb4\xe3\x78\x84\x86\xc3\x21\x9a\xe7\x09\x19\xa7\x39\x49\x34\x03\x37\x5b\xa1\x43\x60\x61\x85\xfc\x09\xc5\x0f\x52\x1b\xa0\xb8\xc8\x39\x2d\x32\x86\x70\x9e\xc0\x03\x4e\x73\x42\xd1\x98\x16\x33\xf4\x06\xe4\x60\x84\x29\x43\x5c\x69\x8d\x68\x4f\x11\xaf\x92\x40\x39\x64\xab\xc4\x7c\x7a\x4e\xc9\x38\x5d\xb4\xfa\xe8\xfc\xe5\xd5\x9b\xdb\xf3\x8b\xd3\xd7\x67\xbf\x76\x64\xf5\x68\x9e\x66\xc9\xcf\x84\xb2\xb4\xc8\x5b\x7d\xf4\xd3\xc7\xb3\xb7\x27\xb7\x3f\x9f\x5e\x5c\x9e\x7d\x78\xaf\x85\xeb\xd3\xdf\xe7\x84\x2e\x23\xb2\xe0\x24\x4f\x42\xa3\x3f\xec\xd9\xb4\x0d\x1d\x6d\xdd\xf0\x34\x7c\x37\x67\x1c\xc7\x53\x12\x51\x92\x27\x84\x86\x8e\xaa\x33\xba\xa8\x5d\x75\x27\x59\x84\xcb\x52\x8c\xe3\x42\x6b\xeb\x05\xfe\x1b\xe1\x88\x92\x31\xa1\x24\x8f\x09\x43\xbc\x40\x38\xcb\x10\x9f\x12\x94\xe6\x9c\x50\xc2\x78\x9a\x4f\xb4\xc6\x62\x28\xcd\xa1\xae\x22\xaa\xa4\x23\xce\x13\x09\x6e\x94\xe6\x09\x22\xf7\x24\xe7\x4a\xbd\x50\xe0\x17\xa3\x96\x7f\xa1\x02\x1d\xaa\x59\x81\x64\xd1\x38\xcd\x93\x30\xf8\x06\x6a\x6f\x1f\x64\x75\x80\xf6\x35\x43\x55\x53\xf9\x87\xa0\xda\xeb\x82\xce\xd0\xd0\x81\xa5\x20\xc8\xfa\xdb\x71\x41\x67\x81\x99\xdd\xcb\x39\x2f\xba\x94\x30\x21\x1c\x02\x6f\x4e\x16\x1c\x61\x4a\x30\x2a\x72\x24\x39\xaf\xa0\x68\x56\xcc\x19\x89\xb3\x34\xbe\x53\xa8\xca\x1e\x57\x64\xc1\xa1\xad\xa3\xf6\x35\xf3\x01\x77\x8c\xc7\x8c\x70\xd0\xe8\x91\xfc\xff\x1b\x92\x4e\xa6\x1c\x75\x45\x49\x9c\xa5\x24\x57\x25\x03\xe8\xf3\x54\xf4\x8f\x62\xc6\xc2\xd6\x14\x8a\x5b\x1d\xd4\xc2\x73\x5e\xb4\xea\xa5\x24\x8b\x58\x4c\x8b\x2c\x53\x00\xf7\xd5\x58\x5a\x45\x9b\xf5\x5d\x94\xd4\x4f\x0f\xae\xb0\xbf\xce\xf1\x8c\x0c\x45\xbb\x9b\xc0\xe2\x8b\x45\x49\xa3\x3b\xb2\x2c\x29\x61\x2c\xac\xa6\xa7\x67\x17\x17\x39\xe3\x88\x08\x16\x10\x52\xf5\xad\xc4\x5f\x88\x26\x89\x1e\xa6\x69\x3c\x45\xc3\xa1\xaa\x7e\xf6\x0c\x3d\x21\x11\x9b\xa6\x63\xfe\x9f\x64\xa9\x01\xd4\x17\x2d\x62\xf3\xd1\x2c\xe5\x61\x7b\xa0\xaa\x49\x54\x52\x60\x94\x13\xa9\x5e\x74\xcd\x4a\x51\x0a\x36\xa4\xa8\xc8\xc3\xd6\x1d\x59\xce\x4b\xb9\x5a\xad\x0e\x4a\xc8\xa8\x98\xe7\x31\x09\x1b\xfb\x19\xaa\xad\x5b\xb5\xa7\x21\xb4\xea\xf8\x4c\x00\x29\x28\xab\xb6\x4b\xcf\x08\x58\xc1\x43\x94\x75\xe0\x9b\x00\x60\x23\x96\x22\x67\x6f\xcd\x56\xbb\x84\x24\xf3\xf2\x27\x9e\x6b\x49\xa8\x08\xa5\xd8\x19\x1a\xdc\x8e\x78\x6e\xaf\x5a\x8e\x47\x19\x39\x11\x35\xeb\xfa\x01\x99\xe4\x9a\x03\x04\x7b\xd1\x4b\x4c\xc5\x4e\x71\x41\x58\x59\xe4\x8c\x6c\x1a\x5d\x35\xbd\xa5\xaa\x6d\x0d\x91\x1a\xa4\x1d\x90\xa9\x03\xb4\xf1\x82\x8d\xe0\xcc\xde\x22\x36\x00\xb2\x76\x0d\x1b\x86\xd0\x8b\x77\x24\xd9\x34\x27\xd5\xa4\x36\x15\x55\xba\xc3\xc8\xaa\xa5\x3d\x6a\x9a\x33\x42\xf9\x3b\xc2\x69\x1a\xaf\x83\xc0\x48\x46\x62\x05\x42\xb6\xbf\x9d\x41\x07\x87\x04\x64\x4c\x09\x9b\x9e\x09\x89\xba\xc7\xd9\x2e\xb0\x54\x97\x9b\xc0\xe2\x2a\x21\xb6\x45\x46\xae\x60\x2f\xf4\x29\x49\xd5\x20\xa8\x6d\x30\xa2\x03\x5a\xd3\x45\x6a\x66\xa3\xeb\xed\xe1\x38\x1e\x31\x7f\x2f\x7c\x2d\x0c\xc4\x2e\x2f\x26\x93\x8c\x0c\x5b\x1c\x8f\x5a\xf6\x74\x45\xc7\x88\xfc\xa3\xb1\xcf\xb7\xc5\x4f\x18\xb0\x69\xf1\x50\x6f\x5d\xe4\xb2\x3c\x8f\x46\xd0\x34\xe8\xa0\xa6\x84\x0a\xa5\xcc\x31\x9d\x80\x52\x7e\x1a\x92\x48\x3e\x28\x25\xe0\xb1\x17\x64\xbd\xe0\x67\x92\xf3\xb0\x1d\xa5\x79\x42\x16\xa1\xdd\xde\x96\x5f\x5d\x21\xf4\xe0\xd3\x30\xf8\x46\xec\x53\x0a\x02\xe6\x9c\x86\x01\xa6\x29\xee\x6a\x5b\x23\x68\xb7\xa3\x29\x66\xaf\x32\xcc\x58\x18\x50\x92\x15\x38\x09\xda\x35\x05\x29\xd5\x22\x58\x04\xb6\x06\x5c\x99\x2d\xec\x82\xf0\x39\xcd\x91\x30\xd2\x19\x1a\x17\xf1\x9c\xa1\x11\x8e\xef\xc4\x4e\x0d\xca\x3f\xcd\x19\x27\x38\x41\xc5\x18\x49\x58\x62\xc3\x8e\x7c\x0c\x1a\x8d\x60\x69\xee\xc8\x32\x29\x1e\x72\x61\x7e\x52\x80\xed\xa5\x64\xa5\xcc\x60\x4c\x87\x24\x50\x7c\x8f\xb3\xd0\x7d\x6a\xab\x36\x12\xea\x1a\x05\xbf\x6a\x57\xaa\x92\xd2\x62\xcd\xe6\x25\xeb\x82\x76\x34\x4d\x13\x45\x75\xe8\xf2\x80\x69\x2e\xec\x11\x7f\x27\x55\xdb\xec\x06\x8d\x5f\xca\xad\x7c\x3d\x8b\x0b\xa5\x5e\x17\x0c\x2d\x88\x06\x82\xd3\xc5\x6a\xbd\x7c\xb9\x48\xd9\xda\xd6\xcb\x5b\xbc\x48\x99\xd5\x3c\x23\x13\x92\x27\x6b\xd0\x91\x95\xb6\x8e\x2a\xd3\x3c\x27\xeb\x68\xa5\x6a\xed\x7d\xe2\x1e\x67\x97\x1c\xf3\x35\xc2\x09\xf5\xb7\x4c\x34\xb0\xa5\x99\xe4\xc9\x09\xe6\xc4\xdf\xc7\xd2\x83\x24\x4f\x9a\xfa\x57\x75\x16\xe7\x42\x22\x4e\x79\x65\x1a\xdf\x11\x1a\x4a\x66\xca\x8a\x18\x67\xa4\x8f\x5a\x24\x6f\x49\x43\x59\x98\x69\x98\xf7\x51\xeb\xb7\xdf\x7e\xfb\xad\xfb\xee\x5d\xf7\xe4\x04\xbd\x79\xd3\x9f\xcd\x54\x3d\x2f\x8a\x6c\x84\xe9\x79\x86\x63\xb0\x3c\xfb\xa8\x35\x2a\x38\x2f\x74\x3d\x4b\x13\xf2\xd3\xf2\x32\x4d\x48\x1f\x71\x3a\x27\xaa\x74\x5a\x3c\x5c\x15\x09\x5e\xfe\x34\xe7\xbc\xc8\xeb\x55\xaf\x32\x82\x69\xb3\xb0\x60\x0e\x10\x81\xfd\x7f\x15\xb9\x40\xf7\xe3\xd5\x2b\x18\x6f\xd5\xf6\x1e\x4c\x0c\x21\x5c\xa1\xa9\x28\x81\xc3\x96\xf8\xef\x55\x3a\x23\xe7\x40\x8f\x56\x1b\x08\xb4\x0e\x8c\x3c\xbc\xd4\xe0\x08\xc5\x97\x94\xca\xa6\xb0\x65\xb5\x8d\x3e\xfb\x74\x88\xc2\xd6\xb7\xad\x68\xc3\xa4\x09\x62\x5e\x0a\xbc\x2e\x64\x73\x0d\xc4\x28\x11\x76\x69\x36\xc9\x86\xd5\xa5\xa4\xdd\xde\x4b\xa5\x36\x80\x33\x5b\xeb\xb0\x55\xb3\x58\x67\x85\x58\xcf\xad\x4c\x26\x9b\x35\xf9\x4c\x96\xff\x69\x36\xeb\x33\xf6\x7f\x12\xa7\x89\x96\x8c\xe3\x59\x69\x6f\x74\x89\x14\xd6\x9c\x3c\xa0\x93\x06\x53\x99\x1e\xcf\x0f\x0f\x0e\x0e\xda\x15\x7b\x56\x04\x5c\xcb\x9d\xe2\x47\xf2\x22\x22\x19\x23\x4d\x74\xec\xc5\x71\x78\x7f\x07\xe0\xeb\x01\x39\xdc\xaf\x20\x7d\x15\xf3\x6b\xc7\x07\x5f\x66\x04\x38\x57\x5a\x85\x0d\xd6\x15\x8d\xd2\xb8\x30\x16\x63\x65\x43\x4a\x7e\x6c\x45\x93\x6c\x59\x4e\x45\x93\x96\xb5\xf3\xbb\x32\x11\x36\x76\xf4\x0a\x0a\x4e\x12\xb5\xfb\x8f\x78\xde\x2d\x69\x3a\xc3\x74\x19\x98\x23\x90\x00\x6c\xb5\x31\x83\x75\xe3\x29\x89\xef\x6a\xed\x28\xf8\xe9\x1a\x4d\xe7\x39\x34\x26\x89\x6e\xae\xd6\x6c\x1d\x4a\x0e\x98\xc7\x61\xd5\x18\x6a\x33\x66\xce\x24\x56\xda\xf9\xe1\x2c\x4a\x68\x29\x19\x0b\xc7\xda\xf9\x4b\xd3\xd7\x47\x7b\x71\xfa\xac\xb6\xdc\xff\xb8\xfc\xf0\xbe\x5a\x8d\x5e\x0f\x9d\x8d\x2d\x77\xc5\x03\x66\x48\x8d\xd2\x81\xe2\x82\xa6\x93\x34\xc7\x19\x62\x84\xa6\x84\x21\xf0\x69\x4e\x0a\x8e\x66\x73\x8e\x39\x49\x2a\x38\x21\x13\x9a\x25\x69\x83\xfb\xe8\x81\xa0\x9c\x90\x44\x58\x60\x94\xc0\x41\x99\xce\x63\x8e\x52\x2e\xdd\x49\x0e\x64\x81\x11\xc0\x8d\xec\xf5\x50\xce\x53\x69\xdc\x52\x9c\x33\xa1\xa7\x4e\x84\xd0\xd4\xe6\x62\x1f\x89\x1b\x1a\xb6\x41\x8b\x1f\x51\xeb\xa0\x85\xfa\x42\xe9\x6a\x73\xad\x4e\x6d\x03\x48\x2a\x7c\x70\xf7\x85\xf6\x21\xb6\xd7\x43\x70\xbe\xcc\xd2\x18\x0b\xea\x57\x96\x24\x83\xf2\x53\x38\x82\x7a\x77\x02\x77\x2f\xb0\xce\xaa\xde\xfd\xc0\x92\xd1\x93\xea\x2c\xec\x01\x5a\x97\x52\x7d\x72\x5e\x2b\xa3\x0e\xa7\xd8\x48\xdb\xa2\xfa\x18\xf9\x7b\xb4\x0c\x36\xa4\xf0\xb1\x72\xf5\x28\xd9\xaa\x4b\x97\x26\x67\xe8\x71\x37\xac\x93\x2d\xef\x92\x79\x49\xb8\x8e\xcb\xec\x51\x6d\x6e\x3a\x97\x07\x7f\xa4\x0f\xfe\x16\x43\x9d\xbb\x5e\x84\x0d\xac\x65\x33\x56\xcd\xf7\xb0\x8d\xb9\xce\x7d\x4e\x8f\xad\x5b\x41\xd3\x55\xb2\xcb\x96\xe0\x9f\x90\xcd\x76\xff\x0e\x95\xff\x6f\x50\xe0\x4d\xa2\xda\xcc\xe6\x21\xde\x26\xb6\xf3\x2d\xe8\x16\x82\xae\x63\x42\x3f\x5e\xae\x87\xae\xe1\x92\xf4\x63\x55\xb9\x52\x2c\x5a\xea\x43\xbc\xa5\x48\xb5\xff\x64\x73\x2b\x9f\x17\x61\xdd\xf9\x5f\x31\xfb\x18\x67\x8c\x0c\x8c\x4d\x68\x1f\x04\xcd\xf9\xb6\x39\x27\x69\x48\x8f\xc0\x2a\xd5\x3e\xa7\xf8\x16\x9c\x66\x37\x41\xdb\xb3\x0e\xda\x4f\x10\x53\x82\x19\xb9\x50\x08\xda\x83\x6e\x02\x9e\x90\x1d\x80\x27\xc4\x03\x7c\x57\xd4\x49\x9e\xec\x82\xf8\x69\x9e\x3c\x12\xed\x2d\x80\x35\xd2\x16\xe0\x5d\x51\x96\xb6\xeb\x2e\x58\xbf\x83\x96\x8f\x44\x7c\x3b\x78\x8d\xbb\x0b\xde\xeb\x13\xf2\x9c\x08\x6b\x8e\x1e\xe9\x73\x14\x75\x01\x25\xa5\x38\x26\x05\x1d\xf4\x99\x93\x05\xef\x7b\xe0\x81\x36\xee\xa0\x59\x21\xce\x4b\xc1\x88\x8c\x0b\x4a\x82\x55\xc3\x7b\xa4\x9d\x4a\x62\x87\xa0\x04\x9e\xd2\x7c\x52\x49\xaa\xbc\x85\x11\x6a\x49\x2a\x6c\xcf\x61\x51\x7b\x41\x45\x23\x75\x42\x34\x3d\x36\xea\x1b\xd9\x6a\x93\xb4\x19\x77\xaa\xd0\x80\xe2\x08\x73\x42\xd3\xb1\x72\x60\xf5\x7a\xc8\xba\x71\x85\xb5\x42\xd3\x94\xf1\x82\x2e\xd5\xc9\xed\x09\x9c\x43\x2f\x79\x41\xf1\x84\x44\x13\xc2\xcf\x38\x99\x85\x81\x6a\x54\x79\x00\x9d\x66\xac\xde\xac\x03\xb6\x63\xc4\x38\x4d\xf3\x49\x3a\x5e\x86\xd7\x37\x6d\xf7\x88\x54\x16\xe5\x3c\xc3\x9c\x9c\x01\xfd\x85\x66\x94\x6b\xc0\x94\x66\x30\x1b\x93\xe5\xa1\xb3\xe9\xd0\x50\x3d\x2b\xff\x15\x79\x75\xd5\xec\xd2\x63\xdd\x3e\xca\xdc\x0b\x67\x59\x38\xa2\xc5\x03\x23\x54\x74\xb6\xcf\xac\x6d\x41\x1f\x51\x18\xb6\x51\x4f\xc5\x5d\x88\x2e\x4f\x23\xfc\x09\x2f\xc2\xca\x7e\x12\x28\x15\x49\x1f\x05\x7f\x3b\xbd\x0a\x3a\xa6\x78\x4e\x33\xe7\x1a\x16\xed\xa3\xa0\x87\xcb\xb4\x77\x7f\xd8\x83\xb5\xf9\x11\x7e\x87\x1c\x86\xb0\x3a\x0a\x13\xfc\x6a\x59\x0a\x26\xfd\xc4\x8a\xdc\xaa\x01\xfa\xcc\xe3\x98\x30\xd6\xaf\x26\x28\x1a\x75\xe0\xfe\xf0\x92\x63\x3e\x67\xae\x25\x29\x89\x2d\xda\x08\x0b\x9d\xcf\x19\x7a\x32\x1c\xa2\x40\x81\x09\xea\x8d\xab\x25\x98\x16\x0f\xa7\x94\x16\x34\x0c\xe0\x8f\xe4\xa7\x34\x9f\x80\x6f\x20\x72\x0d\x42\xf9\x4f\xf2\xab\x5b\xbe\x72\x9e\xe4\x1a\xd0\x7b\x43\x6d\xc0\x0b\x0e\x21\x94\xb0\x79\xc6\xaf\x0f\x6e\x06\x8d\x1e\x49\x3a\x16\xab\xf6\x0e\xf3\x69\x84\x47\x2c\xb4\x17\xac\x6b\xc1\x93\xbc\xe5\x4e\x1c\xfa\x1e\x0f\xd1\xb7\x07\xcd\x99\x42\x68\x88\x98\xe7\x2f\xd2\x3b\x1b\x36\x66\x84\x50\x70\x94\xa4\xf7\x28\x16\xbb\xe7\xf0\xf7\x00\x67\x84\x72\x04\xbf\x5d\xe5\xd2\xfd\x3d\x38\x3e\x62\x9c\x16\xf9\xe4\x58\x81\x79\x72\xd4\x53\x05\xe8\x84\x70\x12\x73\x92\xa0\x00\xed\x7b\x80\x0b\xe4\x22\x5e\xbc\x4e\x17\x24\x09\x5f\xb4\xbd\x6d\x02\xc4\xc4\x49\x2e\x61\x40\x77\xe8\x22\x6f\xc4\xd1\x88\xf0\x07\x42\x72\xb4\x2c\xe6\x86\x89\xe1\x14\x28\x8e\x79\x92\x2a\x91\x1d\x87\x44\x49\x26\x8e\x92\x45\x8e\x70\x1c\xcf\x29\xe6\x44\x82\x84\x2e\x00\x1b\x44\x67\x06\xb7\xba\x31\x9e\x33\x82\xe6\x39\x59\x94\x72\x06\x52\x9d\xc8\x55\x62\xd1\x51\x2f\x49\xef\x8f\x83\x1a\xbe\xed\x75\x6b\xbf\xaa\x78\x18\xdc\xe7\x7d\xdf\x69\x4a\xfe\xf3\x33\x9f\xb0\x5a\xbc\xbc\x27\xc7\x58\xad\x8b\xea\xa9\x14\xc4\x5a\x95\xb4\x53\x68\x4a\x4d\xe8\xbd\x22\xbf\x49\xe0\x33\x3c\x22\x59\xef\xf6\x56\x6c\x0c\xb7\xb7\xbd\x7b\x08\xeb\x31\x3d\xd7\x49\xfc\xe3\x64\xfd\x11\x72\xbe\x99\xc8\xf8\x1e\xa7\x99\xa0\x10\x92\xb7\x81\xec\x89\x2b\xed\x75\x39\x5f\x55\x62\x57\xe2\x09\x79\x55\xe4\xe3\x74\x12\xe1\x2c\xab\x28\x6c\xe4\x1c\xb6\x55\x5e\x24\x45\x1f\x25\x85\xf1\x57\x00\x3e\x55\x87\x1f\xd1\x07\x8a\x62\x9c\xa3\x94\xa3\x4f\x73\xc6\x51\x96\xde\x13\xc1\xb8\x82\xb3\xc5\x10\x66\xbc\x71\x41\x51\x08\x27\x24\x88\x46\x42\x29\x3a\xf2\xe3\x10\x65\x24\x9f\xf0\xe9\x00\xa5\xfb\xfb\x1e\x5a\xd8\x86\xc2\xf5\xc1\x8d\xf1\x02\xe2\x24\x09\xc5\x8e\xf0\x01\x9e\x43\x2f\xe8\xeb\xf4\xa6\xe3\x1f\xf4\x3a\xbd\x69\xb7\xbd\x74\x82\x41\xc7\xf3\x3f\xfe\x58\x5e\x80\x44\x99\x28\x1e\xf9\x0f\x84\xad\x0f\x61\x6d\x1d\x87\xf0\xa2\x6d\xb3\x7c\x86\xcb\x3e\xfa\xbc\x5a\x3b\x90\xb0\x0a\x04\x7f\xe1\x29\xc1\x32\xdc\xa6\x3a\x9f\x6b\x38\x9b\xe4\xf2\xeb\xd9\x65\xa5\x3d\xc7\x5b\xa4\xd3\xc1\xd0\x96\x48\x40\x16\x50\x91\x71\x1f\xac\x98\xd3\x58\xec\x1c\x40\xa2\x37\xd2\x22\x89\x52\x66\x9f\xbf\xac\xb5\x30\xad\x34\x1b\xc4\x45\x1e\x63\xee\x5f\xc8\x36\xea\xfb\xd7\xd1\x8d\x4d\xe1\x86\x92\x92\x42\x78\xce\x8b\x4b\xb0\x44\xfb\xd2\x56\x53\x7e\x75\xc0\xb4\xaf\xfe\xca\xb2\x94\x93\x19\xeb\x83\x31\x21\x0b\x66\x98\xc7\x53\x62\xd1\x1d\x85\xa2\x4d\xdd\x53\xf8\x40\xd0\x14\xdf\x13\xc5\x00\xc0\xf5\xf1\x9c\x52\x92\x73\x49\x87\x0e\x62\x77\x69\x59\x77\x31\x59\xfc\x25\x09\x01\x2a\x01\x76\x3d\x78\x6c\x2c\x71\xb3\x83\xdd\x7c\xb0\xbe\xf1\x0c\x97\x82\x83\x57\x1b\x9a\x50\xcd\xe7\x50\x18\x8d\xd3\x8c\x13\x1a\x56\xd0\x23\x65\xc1\x87\x3d\xd4\x9b\x74\x50\x10\xb4\x3b\x6a\x83\x96\xf4\x73\xe4\xa3\xa4\x42\x57\xea\x7d\xd7\xb1\x90\xca\x82\x71\x51\xa7\xf7\xe0\x6a\x8f\x5a\xb5\xb7\xa2\x17\x8d\x0b\x7a\x8a\xe3\x69\x65\x9e\x53\x8f\xb2\xa8\xcd\xfc\x9a\x46\xda\xa9\x7a\x83\x86\x88\x0e\x3c\x23\x1a\x89\x54\x36\xbd\x58\x64\x94\xe6\x5e\x78\x3a\x2c\x68\x4f\xb1\x11\xe5\x4d\x06\xb1\x14\x3f\x3c\x46\xa2\x59\x85\x35\xee\x8c\x6c\xbc\xb5\x82\xf4\x62\x3f\xba\x89\x58\x5c\x50\x69\x4a\x79\xea\xb1\xaa\xaf\xa6\xa5\xe7\x00\x9e\xad\x03\xf4\x23\xc2\x91\xbc\xdf\x7a\x55\xcc\x4a\x4c\x49\x38\x12\x92\x94\x9a\xb9\x1b\x2a\x58\x93\x67\x6e\x40\x01\x30\xfa\xd5\x34\x65\xb0\x1f\x40\xb0\xdf\x14\xa2\x03\x11\x1e\x73\x61\xd6\x70\x8e\xe3\x29\x58\x00\x53\x82\x8c\x04\xa2\x32\x9b\x4f\xd2\xbc\x83\x30\x43\x29\x97\x50\x0a\x3e\x25\xf4\x21\x65\x04\x8d\x28\xc1\x77\xac\xd6\x43\xd3\x08\x67\x29\x5f\x46\x7b\x6b\xc2\x09\x1c\xed\x32\x4a\xf3\x44\xfd\xff\xf4\x9e\xe4\x9c\x69\x15\xba\xda\xa8\xd3\x26\x84\x7f\x30\x31\x9a\xdb\x4d\x8c\x5a\x4c\xe7\x6a\xe0\x06\x7a\x42\x10\x92\x8e\x04\x46\x28\xb0\x82\x8d\x14\xff\x07\xe6\xb6\x56\x17\x30\x4e\x4a\xb7\x24\x29\x1e\x72\x86\x67\x65\x46\xea\x2d\xc1\x53\xaf\x1f\xed\xcb\x2f\x21\x40\x37\xeb\x3d\x0c\xb2\x4d\x3b\x22\x8e\xd8\x40\xb0\x4a\x47\x07\x67\xda\x47\x30\x61\x01\x55\x81\xe6\x91\x78\xb4\x22\x57\xa2\x34\x7f\x49\x29\x5e\x86\xa2\xbc\xe3\x4c\xbd\x2d\xcc\x78\xcb\x8a\x87\x88\x3f\x05\x05\xec\x29\xb5\xc5\xa3\x63\xe4\xd8\xfa\x8a\xa6\x70\x26\xbf\xb1\x46\x86\x3e\xb6\x9b\xda\xe6\x46\x73\x01\x2e\x83\x18\x6b\x67\x55\xbb\x85\x0c\xd6\xa9\xc7\xef\xc8\x23\x3f\x70\xbb\x89\x8f\xdf\x66\xa0\x62\xca\xc8\x89\xb0\xcb\xd3\xc2\x71\x04\xc3\x4a\x5f\x91\x05\xaf\x58\x07\x8a\x2e\x4e\xd5\x71\xf5\x82\x4c\x4e\x17\x65\x18\xfc\x77\x78\x7d\xd0\xfd\xe1\x66\xbf\x1d\x5e\x2f\x1f\x92\xe9\x8c\xdd\xec\xb7\x9f\xca\xbd\x58\x74\x92\x7b\x8d\x60\x21\x03\x31\x82\xb2\x50\x81\x33\x17\xbc\x4f\x54\xd3\x36\xfa\xac\x8d\x3d\xa0\x8d\xa8\x53\x55\x9a\xd8\x4f\x86\xe8\xdb\x9a\x2b\xfc\xfb\x03\xed\x0b\x10\xa3\x02\x99\xd1\x10\xc1\xf4\xce\x72\xae\x01\x5c\x1f\xde\x18\xcc\xe6\x79\x2a\x76\x06\x5d\xf3\xe2\xc6\x22\x9f\xec\xff\xbc\x19\x12\x6e\x05\xec\x5f\x0b\x00\x37\x3b\x18\x19\x96\xb3\x6f\x67\x99\x04\xe2\x5c\xaa\x33\x58\xe5\x83\xaf\xd6\x4a\x6e\xb6\x55\xa4\xa0\x15\x71\xe4\x33\x4f\x37\xc4\xf9\xfb\x8c\x54\x41\x73\x07\x85\x23\x1f\x0a\x1b\x80\x82\x11\xea\x5e\xba\xd6\x70\xdd\xd2\xb9\x71\x75\xd5\x74\xda\xa0\x0d\x3e\xe3\xea\x1c\x68\x9f\x1b\x56\xbb\x38\x75\x1c\xef\xec\xbf\x7f\xc1\xb6\xaf\x14\xea\xa2\x43\xb1\xaa\xc7\x72\x75\xbb\xdd\xb5\xab\x76\xfc\xff\xce\xaa\x4d\x08\x3f\x35\xf1\x5a\xdb\x97\x0c\x14\x8e\x13\xe5\xf5\xe5\x0b\x72\x0a\x5c\xac\xa9\x8e\x3a\x54\x0e\x64\xa5\x6b\xdc\x0b\xde\xed\x71\x4e\xdb\xcf\x24\x62\xff\xa6\x97\x8f\x9b\x8c\x15\xfc\x22\xaf\x60\x4c\x77\x2b\xe6\x8f\x55\x85\x26\x9e\x45\xa1\x9f\xc0\x7b\x61\x5b\x10\x63\x5e\x9c\x00\xd4\xc6\x57\x6b\x76\x21\x8b\x42\x68\x47\x4d\x7a\x9a\x7b\xee\x44\xd7\x90\x25\x27\x0f\x0a\x65\xb5\x74\x9a\x40\x36\x91\x95\x18\xaa\xb6\x70\xfc\xde\x59\x7e\x51\x0f\xbd\xe8\xa0\x96\x72\x97\xb5\xbc\xf4\x56\x80\xad\x3a\x97\xf5\x77\x54\x48\xff\xea\x79\xb3\xf9\x88\x53\x1c\xf3\xff\xad\x26\x3f\x21\xfc\x9d\x8e\x90\x7b\x8c\x58\xab\xb0\x3a\x23\xd5\x2a\x7e\xea\xb1\x42\xbd\x43\x00\xd7\xee\x32\xfd\x88\x89\x78\x44\xfa\x9d\x85\xa6\x26\xb2\x2a\xfb\x5a\x81\x6e\x22\xb4\x5d\x9e\x77\x8f\x97\xdb\x51\x9c\x1f\x49\x95\xcd\x9c\xad\x89\xd4\x10\xe8\xc3\x83\x75\x8c\xaa\xba\xfc\x93\x84\xf4\x5f\x3f\x1b\x23\xa6\xff\xea\x29\x59\xad\x77\x7f\x93\x32\xce\x08\xa6\xd2\x63\xd7\x76\x0b\xf5\x7d\x47\xbb\xb6\xff\x36\x2c\x84\x6a\xef\x5f\xed\xd5\x2f\xfa\xd9\xb4\x78\x08\x3d\x81\xdd\x11\x99\x95\x7c\x19\xda\xc1\x8e\x98\xf2\x0d\xb7\x6b\xff\x0c\xbb\x4d\xbd\x1a\x57\x64\x73\x75\x7a\x32\xc7\x8d\xf5\x87\x56\xfd\xea\x8c\x3e\x23\xdf\x04\x6d\x3d\xfb\x2f\x5f\xe4\x6d\xd3\x0c\x2f\x42\xf8\xcf\x38\x2b\x0a\xea\x5a\x74\x3d\xf4\xe2\xbb\x83\x76\x07\x1d\x5a\x07\xac\xc5\x25\x38\xab\x2e\x6c\x4c\xbc\x08\xd8\x6f\xcb\xcc\xf0\xe2\x56\x3a\xb9\x6e\xab\x29\xd4\x50\xd2\x43\x54\x81\xf0\x0d\xf3\xc2\xba\xa4\x04\x64\x34\xf7\xd7\x95\x96\x69\x67\xdf\xc3\x02\x7e\xbf\x4e\xa9\x73\x0b\xab\x0b\x23\x3c\x2a\xa8\xa5\x95\xe1\xc8\x46\x33\x13\xcb\x24\xaf\x18\xf4\x63\x89\x29\x9e\x55\xaf\xa5\x06\x00\x25\xe8\xd7\xcf\xd0\x26\xf4\x5a\xb6\x97\x61\x60\x68\xb8\x26\x3a\x0e\xfd\x88\x5a\x9c\xce\x09\x04\xd6\x80\x93\x55\x8a\x99\xea\x5c\x7f\x85\xcb\x82\xb3\x29\x4e\xa7\x09\x71\xd3\x4b\xbe\xc6\xab\xa0\x06\x05\xbe\x46\x43\xb3\x26\x5d\x87\x83\x07\x76\x53\xf9\x42\x85\x6a\x38\x70\x81\x10\x31\xeb\x6a\xe1\x9d\x5a\x2f\x6b\xc0\x49\xb9\xc1\x69\xb2\xdf\x9c\x66\xe2\xe0\xb2\xe1\xd6\x59\xc6\xc4\x04\x2a\xea\x41\x2e\x9d\xad\x53\x3c\x37\x4c\x76\x9c\x10\x68\x26\x4d\xcc\x66\xe3\x81\x8c\xc4\x72\xc2\xbe\xd4\x5c\xb8\xd4\x00\x92\x2f\x77\xc5\xf6\xab\xf1\x7c\x25\x63\xa4\xb6\x63\x6a\x45\x86\x49\xb6\x95\xff\xa9\x39\xbc\x7e\x9d\x52\x34\x5c\x77\x0b\x58\x93\x70\xf9\xbe\x95\xac\x0c\xda\xce\xed\xe0\x9c\x66\xdb\xee\xfc\x44\x79\x5f\x21\xf1\xef\xbe\x07\x84\x5e\x70\x11\xb4\xe5\xbe\xaf\x31\x94\xba\x0c\x67\x6b\xc0\xeb\x0d\xc7\x6d\xeb\xbd\x19\x73\x3c\xaf\x72\x2d\xd5\x93\x7b\x71\x65\xa8\x12\x9a\xbb\x45\x77\x7d\xb7\x5d\x70\x2d\xa6\xb4\x03\xa1\x9f\x75\xda\x89\x32\xf4\x64\x88\x02\x50\x7b\x35\x8a\x81\x12\xa6\xd4\x26\x8f\xe8\xb3\x98\xd2\x48\x2b\x1f\x08\xac\x7e\xe2\xcb\x0a\xa0\xff\x11\x2a\xb8\xa9\xde\x47\x52\xde\x86\xec\x09\xd5\xb5\x3b\xcb\xf5\xbd\x22\x0b\xee\x74\xda\x7a\xcf\x4b\x16\x24\x9e\xc3\xcb\xf3\xea\x9e\x31\x40\xfb\x02\x6c\xe3\x5a\xdd\xa2\x5e\x5c\xcc\xca\x8c\x70\xb2\x33\x01\x87\x6b\x08\xb8\x9e\x99\xc0\xd0\xae\xfc\x9f\xde\x40\x9c\x6e\x65\x4d\x0c\x9c\x8e\xbc\xe0\x38\x13\xc5\x97\x32\x60\x1e\x72\x53\x6c\x5a\x21\x19\xe9\xbe\x61\x99\xd6\x76\x52\x77\x45\x42\x78\x61\x5f\x08\x58\x8c\x33\x4c\x1b\xa1\x34\x4d\x94\x0e\x3d\x8b\x9b\x8e\x37\x8e\x02\x18\xe6\xf3\x2c\xdb\x0e\x7d\x13\x18\xed\x59\xf4\xf2\xc9\xca\xf5\x06\x55\x96\xdc\x94\xcf\xb2\x30\x78\x5b\x60\x19\x20\x22\x19\xc5\x2c\xd1\x3e\x0a\x66\x0c\x1d\x8d\x28\xea\x1d\xa3\x6a\x23\x92\xad\xac\xed\x6a\x1f\x05\xba\x99\xa8\x09\xae\x04\xe6\x32\xe2\x44\xbe\xdd\x20\x7b\xd4\x26\x54\xbf\xac\xab\xc7\x97\x56\xa8\xef\x70\xbd\x6c\x44\xc0\xde\x41\x66\x6c\xb2\xc5\x5f\x22\x7a\x44\x42\xa7\x40\xdb\x5a\xb9\xb6\x7f\xb7\x05\xa6\x19\x2b\x7c\x77\xcb\xdd\x1a\xb8\xd5\xaa\x8f\xab\x09\xb0\xc3\x94\x7f\x31\xef\xa4\xee\x3e\x69\xa5\x9d\xe5\xda\x3b\xd3\xd6\x35\x8f\x99\xb8\x07\x83\x47\x0c\x6f\x4f\xde\x54\xec\x36\x7d\xe7\xad\xc1\x1d\x86\xb7\x0d\x3f\xc1\x9a\xc5\x9c\x9f\x9d\x68\x99\x7b\x48\xf3\xa4\x78\x90\x33\xba\x92\x95\xf5\x96\xe6\x8c\x94\xd6\xde\x93\xf7\x9d\x60\x6a\xaf\x3e\x56\xc7\x18\x38\x8b\x69\x08\xee\xed\x8b\x79\xe3\x5c\x0f\x89\x86\x1a\x2f\x26\xd5\xa3\xc0\xca\x1f\x56\xea\xf1\xef\x42\x79\xfd\xd5\xca\x3d\xc8\x46\x61\x66\xf0\x5c\xa5\x9d\xda\x4e\x6d\x99\xf3\xe5\x2d\x1e\x91\xcc\x31\xd2\x20\xec\x89\x55\x24\x87\xe7\x4b\x88\x1b\x65\x2a\x45\x93\xe5\x73\x87\x5a\x94\xe6\xc8\xee\x26\x89\x22\xab\xc4\xa6\xac\x63\xa8\x2c\x75\x6b\x43\x8d\xca\x39\x9b\x86\x55\x68\x00\xda\x57\x60\xf7\xad\x98\x00\xb5\xe3\xb1\x18\x97\xe4\xcd\xd5\xbb\xb7\x0a\xcf\x6b\xf8\x63\x42\x76\x56\xae\x13\x2a\xd3\xb3\x73\x43\x02\x65\xf1\xef\x41\x35\x94\xc6\xe4\x53\x91\xe6\x61\x70\x34\xa2\xc7\x41\x5b\x0e\x0f\x31\x73\x5b\x89\x29\xa3\x68\xae\x8a\x2b\xf6\x5e\xde\x95\xae\x25\x27\xd7\x2d\x54\x4d\xa4\x89\x23\x0e\xb0\xad\x16\x8c\xfa\x39\x18\x6c\x22\xfe\x56\xea\x6f\x27\xbf\x87\xfe\x86\xe4\xc3\xdf\x03\x43\x17\x4d\x5f\x51\xfe\x7b\x60\x62\x84\x60\xf7\x11\x3f\x6a\x36\xfb\x43\x1f\x19\x3b\x92\x86\xab\xc0\xf2\xad\xc9\x0e\xbb\x5d\xac\xfe\xac\xae\x21\x0d\x2d\xe1\x5e\xb1\x22\xa5\x94\x58\x68\xfa\x3a\x2b\x30\x57\xf5\x5a\x28\x53\xf6\x1e\xbf\x17\x65\xc6\x35\xd2\xeb\xa1\x60\xff\x2c\x1f\x07\x1d\x14\x74\xd5\x5f\x78\x46\x0f\x69\x96\xa1\x11\x91\xc0\x12\x21\x4e\x05\x7a\x8f\xdf\xa3\xd1\xd2\x86\xdf\x8e\xd0\xd5\x94\x68\x50\x31\xce\x5b\x5c\x74\x82\x68\x72\x92\x74\x10\x2b\xe0\xcd\x5c\xc4\xa7\x64\x86\x30\x43\x13\x5c\x32\x14\x82\x25\x10\xd9\x2e\x53\x9d\xfa\x6c\xe5\x5c\x99\x6e\x25\x8a\xf3\x6e\x5f\xfd\x5c\xb5\xd1\x4f\x56\xe2\x8c\x70\xf3\x7a\xef\x85\xca\xc4\x16\xbd\x2a\xb2\x82\x46\xe7\xb2\xb2\xf2\x2c\x81\x71\x6e\x19\x4c\x82\x87\x66\x98\xd3\x74\x11\xb8\x2a\xaa\x32\x52\x55\x44\x5c\xca\x50\x5e\x70\x54\x8c\x91\x6c\x0f\x31\x1c\x4f\xd0\x79\x46\x30\x23\x2a\xc1\x0f\x46\x71\x41\x29\x89\x39\x24\x9c\x20\x8c\xa5\x45\x6e\xc2\x43\x15\x35\x24\x9f\xaf\x2a\x4f\x2e\xd6\xf1\x88\xd4\x04\xb9\x54\x7a\x93\xb3\x7a\xb0\xc2\xc0\x3c\x49\x2e\xae\xa2\x15\x38\x53\xb2\x0a\x66\xa0\x6b\xa4\xa9\x30\x07\x6d\x1b\x0e\x6c\x55\xc5\xac\x58\xa9\x9a\x89\xaf\xa3\x23\x2a\xd5\x24\x3d\xbd\x8e\x4a\xa8\x06\xae\xa2\x0b\x0d\x60\x53\x57\x29\x31\x43\x0a\x7b\x94\x3e\xfc\x76\x9c\xee\x7d\xf5\xd7\x3d\x8b\x72\x26\x63\x25\x98\x4b\x29\x4b\x80\xe4\xbf\xda\x20\xe2\xdf\xa2\x2f\xef\xef\xaf\x0f\x6e\xec\x18\xad\x65\xdf\xda\x1b\x41\x32\x25\xb4\xeb\xc3\x9b\x76\x65\x95\x56\xf1\x43\xd5\x21\x24\x13\x47\x38\xc5\x81\x11\x3c\x86\xb2\x87\x3c\xcc\x03\x39\xc0\xec\x6d\x04\x72\x31\x4b\x70\x65\x0c\x30\xac\x18\x03\x05\x88\xb3\x0c\xcd\x52\xc6\x84\xa9\xc2\x38\x29\x59\x54\xb1\x00\x79\x30\x16\xb6\x52\x99\x52\x0c\x0a\xeb\x90\x61\x94\x28\xb7\xb6\x7d\xe3\x23\x1a\x20\x8e\x8e\xdc\x72\x92\x27\xa2\x74\xbf\xde\x9a\x94\x4e\xe8\xdf\xcb\x2c\x2b\x1e\x00\xfa\x58\x28\x0d\x81\x5e\x59\xa4\x39\x47\x69\x2e\x63\xb8\xe3\x65\x64\xdf\xf3\x4a\x93\xdf\xc4\xc1\x08\x1c\x9f\x3d\x43\xb2\xf8\xba\x2c\xd8\x4d\xb4\x40\x47\x62\xdc\xc6\xb0\xd2\x2b\x68\x2f\xa7\x99\xb8\x54\xe9\x16\x10\xcb\x34\x2f\x0b\xc8\xc8\xa7\x16\xaa\x7e\x5c\xad\x81\xf8\xbc\xe8\x23\xde\x41\x2a\xb0\x75\xd5\x6e\x06\xdf\x20\x64\xd2\x37\x9a\xbe\xd5\xc2\x56\x57\x2a\x78\x47\xfb\xaf\x91\x1b\x73\xe3\x75\x15\xd0\xc4\xa6\xa0\xf6\xfa\xb9\x66\x18\x64\xbe\x81\xcc\x95\x38\x5f\x22\x4e\x71\x4c\x98\x50\x53\x38\x47\x64\x91\xca\xac\x74\xa0\xc6\x23\x37\xa5\x4a\xe5\x18\xb7\x86\xab\xf2\xb1\xc4\xd3\x34\x4b\x28\xc9\xc3\xb6\x27\x90\xa9\x6a\x5b\x7b\x47\x08\x2a\x20\xc3\x8b\x53\xb1\xaa\xa7\x8a\x79\x1a\xb6\x2c\xb3\x25\x90\x39\x62\x8e\xa5\x49\x62\x59\xd9\x3a\x57\x4c\xad\xb9\x4a\x12\xd3\x6c\x5f\xa1\xdf\xc8\xe5\xb7\xad\x11\x0c\x55\xdd\x12\x90\x3c\x51\x77\x04\x6b\x3d\xdb\x82\xf2\xaf\x8a\xfc\x5e\xc8\x2e\x2f\xd0\xc7\xf7\x67\xbf\x22\x93\x4c\x42\xe7\xf2\xb3\x3c\x08\xbb\x5f\x9e\x7e\xf9\x82\xbe\xfd\x5e\x8d\x70\x38\xd5\x69\x25\x23\xcf\x05\x86\x46\xb3\x6b\x06\x32\xd3\xdc\xae\x77\xce\x71\x02\x11\xd3\xea\xa5\xfe\x87\x94\x4f\x51\x9a\xdf\xa7\x2c\x1d\x65\x04\x05\x42\x2a\x02\xa9\x30\x19\xc2\x1c\x02\x17\x63\x88\x45\x9e\x53\x92\xa0\x45\x57\x2c\x02\x1a\x15\xf3\x3c\xc1\x00\x80\xe4\x6c\x4e\x09\xd3\xe0\xf9\x14\x73\xc9\x79\x0c\x61\x4a\x50\x92\xb2\x32\xc3\x4b\x92\xc8\x91\x30\x1a\xa7\x8b\x0a\x0e\x50\xc1\xc9\xd1\x94\xe3\xb2\x84\x10\xcb\x02\x86\x36\x71\xdd\x06\xbe\x98\xb8\xee\x06\x4d\xaa\xf4\x01\x95\xfa\xb9\x3e\x10\x5a\xe6\xb8\xa2\x9a\x15\xc6\x22\x69\x34\xcf\x21\x25\x1f\xe8\x03\xd3\xaa\xa1\x17\x56\x75\xb8\xae\x76\xeb\xa2\x43\xa9\xcd\xd4\x8a\x34\x46\x31\x2a\x47\x35\xf0\x0e\x50\xbd\x0c\xfe\xbe\x78\x40\x31\x25\xf0\x56\xcc\x94\x80\x6d\xe3\x0a\x71\x23\xe7\xac\x6d\xfd\xc8\x6c\x05\x12\x03\x15\x60\xd8\xb7\x98\xdf\xec\x7f\x32\x8b\x62\xbf\xba\x5d\xb2\x04\x1b\xfc\x1b\x32\xa9\x62\xd8\xee\x80\x3a\xee\xa8\xe3\x67\xc2\xa7\x1b\xfa\xfc\x22\xea\xc1\x39\xf6\xd7\x83\x0e\x7a\x61\xfa\xc9\x53\x19\xa1\x7d\x4f\x72\x8a\x1f\x55\xcc\x67\x80\xfa\x28\xc8\xd2\x9c\x68\x4f\x35\x9c\xfe\xca\x22\xc3\xca\x97\x23\xea\x30\x55\xee\x69\xed\xaf\x31\xfc\xae\xa2\xd8\x53\xd1\x12\xcf\x79\x11\x74\x1c\xa2\xbe\x4e\xf3\x04\xde\x2f\x62\x44\x71\x66\x8b\xa1\x19\x5e\xf4\x66\x69\xbe\xb7\x26\x6d\x86\x50\xba\x9c\x56\xa6\x45\xaf\x87\x7e\x99\x92\x5c\xe7\xc7\x10\x76\xa1\xcc\xdd\x96\x98\xbd\x78\x86\x17\xd5\x5e\xbc\x41\x16\x79\xe5\x5d\x72\x72\x38\xc4\x73\x4a\x65\xf9\x3b\x1b\x92\x4c\x83\xa3\x76\x30\x3f\x44\x51\x7a\x2e\x76\xe4\xba\x0f\xd4\x54\x44\x4b\x74\x5c\x1b\xe0\xd9\x33\x64\x57\x3f\xf1\x39\xf8\xea\x28\x59\x1d\x3c\x5e\x5a\xb3\x95\x0a\x4a\xec\x0f\xdd\xde\x8a\xdb\xed\x0d\xc3\xe1\xe5\x48\x92\x6f\x86\x17\xcf\x0f\xa3\x83\xef\xd6\x37\x4b\x73\x4d\x1b\x67\xa7\x87\x15\x80\xba\xb3\x7c\x9c\xe6\x29\x5f\x0e\x6a\x2b\xd3\x75\x2b\x1e\xb9\x42\xff\x9c\x45\x38\x02\x1c\x77\x21\xbd\x9c\xcb\x46\x82\xfb\xd6\x78\xb6\xe3\xca\xce\x76\x5f\xcf\x95\x95\xc7\x01\xb0\x1a\xc2\x32\xd5\xe3\x02\xfd\x8b\x89\xf6\x2b\x7f\xf3\xda\xd5\x14\xbf\x5d\xdd\xce\x97\x9f\x67\x3d\xf0\xf0\x20\x3a\x7c\x1e\x9a\x97\x32\x45\x61\x57\xc0\x6b\x57\x87\x92\x2d\xc3\x6e\x85\xb0\xd2\x4e\x35\xc1\x4a\x0b\x65\x9a\x34\xf5\x6e\x04\xe6\x0f\xdc\x10\x7c\x96\x5a\xa6\xef\x53\xd9\xd6\xfb\xdb\xcb\x2d\xb0\x7e\x53\xaa\x7c\x2d\x30\xa9\xf7\x0a\x9a\x92\x9c\x1b\x4d\x49\xc6\x3a\x9e\x9e\xa7\xf1\xdd\x6b\x95\xe0\x0b\x5e\x62\x91\xd9\xbe\xfe\xf3\xdd\x4f\x57\x1d\xcf\x1e\x01\xe8\xa8\x3d\xc2\x7e\xc9\xdb\x25\x9d\x4a\xac\x5c\xcd\x62\x5a\xdc\x13\x7a\x42\x38\x4e\x33\xff\x5c\xde\x54\x0d\x76\x9b\x90\x44\xd3\x79\xdf\x24\x94\x3a\xbf\x83\x16\x1d\xb4\x74\xd5\xa6\x8a\x8a\x6a\x1d\xb1\x12\xe7\xda\x54\x14\x85\xc1\x71\x0b\xed\x57\x17\x38\x0b\xf4\x1c\x0c\xb8\x76\xc4\x8b\x8f\x57\xaf\xa4\x63\x27\x6c\xa3\x7d\xd4\x3a\xea\x89\xbe\xc7\xad\x81\x05\x96\x3d\x60\x1e\x4f\x9b\x80\x61\x1e\xb7\xb2\x36\x90\x49\x35\x86\xc1\x08\xc7\x77\x13\x2a\x4c\xa2\xae\x3a\x1d\xb6\xe0\x74\x03\xea\x02\x4a\xc4\x30\xc2\x72\x6d\x0e\x14\x17\x39\x57\x31\x12\x72\xc8\x7d\xa4\x66\x1b\xf9\xfc\x69\x60\x98\x49\xa7\x5a\x1f\xd9\x0e\xc6\xa5\x9a\x89\x2c\x31\x43\x58\x11\x60\xd0\x60\x44\x81\x2c\x7a\x54\xab\x48\x79\x85\x2b\x1f\xaa\x8b\x46\xd3\x5e\x01\x6f\x84\xce\xd3\xe8\x59\xf8\xb7\x50\xe7\xb5\x47\x64\x37\x63\x90\x6c\x64\x08\x6b\xb4\x69\x3a\x99\x66\xc2\x34\x81\x5c\x8f\x9e\x21\x7f\x22\x53\x7c\x9f\x16\x34\x52\xaa\xfa\x8d\xee\x10\xa2\x9d\x58\x4f\xe2\xd5\x57\x7f\xdd\xc1\xd9\x94\x64\xf7\xf2\x1a\x61\x87\x91\xaf\xc0\x3a\xd8\x8d\xe1\xd7\x8d\x6a\x47\x12\x98\xcc\x28\x5b\x9d\xe0\x2c\xfd\xe3\x6b\x8e\x9c\xae\x9a\xaa\x5f\xf8\x79\x34\x81\x39\x14\x98\x50\x84\xaf\x35\x11\x37\x58\x05\x95\xba\xd9\x21\xe6\xdb\x13\x1c\xb2\x25\x58\xc3\x4f\x13\x71\xb6\x56\x58\xa8\x54\x67\x0c\x95\x98\x31\x78\x47\xb8\xca\x84\x36\x2e\xa8\xb1\x07\xe5\x81\x07\x1c\xa6\x56\xfa\x33\x86\xef\xc9\x9e\x3a\x15\x59\x49\xcf\x5e\xfe\xc7\xcb\x5f\x4d\xbe\x27\x71\x8a\x29\x68\x42\xa8\xcc\x97\xd6\x35\x3e\x51\x94\x72\xe9\xb6\xb5\xc6\x94\xc0\x1e\x84\x25\x2a\x20\xce\x19\xa1\xe2\x80\x25\xce\x47\xf2\x15\x33\xc0\xc7\x4e\x70\x6b\x72\xa5\x29\x7f\xa3\x73\x50\xf4\xe7\x58\x03\xe7\xeb\x56\x77\x84\xd7\x6b\xfa\xbe\x00\x34\xc1\x3d\xc4\xd0\x58\x68\xc4\x9a\x27\xb4\xe9\x17\xb8\xc2\x23\x37\xc3\x92\x9d\x1e\xc8\xba\x21\x32\xb9\xd8\x76\xe2\x82\x5a\xe8\x4d\x2d\xa6\x15\xef\xc4\x07\x32\x56\xb1\xca\x73\xb4\x19\x4b\x9b\xd2\xd2\x1f\xae\x2f\x48\x7e\x2a\x92\xa5\x26\xb5\x05\xce\x4d\x39\x7d\x0b\xe9\x00\x10\x1f\x15\x89\x4a\x36\x08\xfd\x9c\x40\x46\xf6\x90\xf2\x78\x1a\xd6\xee\xff\xd5\x9b\xc9\x98\x11\x14\xdc\x93\x98\x17\x34\xe8\xef\xd9\xe6\xa1\x79\xed\x55\x09\xb5\xd8\x48\x9a\xd7\xf2\xee\xca\xea\xe1\x95\xb3\x24\x38\xe2\xf4\xf8\x88\x27\x28\x2e\x32\xb1\x87\x0d\x5b\x2f\x5a\xc7\x47\xe9\x71\x2e\x17\xfc\xa8\x97\x1e\x1f\xf5\x78\x22\x7e\xe8\x71\x50\x7b\xd7\xd2\x3e\x4a\x3b\xf8\xa8\xcd\x20\x9d\xa5\x5c\xaa\x4d\x1b\xdd\x76\xcd\xab\x69\xbd\xbd\xe4\x89\x29\x70\x5f\xa9\x87\xc5\x54\x06\xae\x4e\xea\x91\xde\xd8\xdb\xae\xb9\xb5\xf2\xb9\xb6\x8d\x67\x7b\xb0\x89\x16\xc7\xb5\xfb\x3b\x09\x52\xdd\xb2\x09\x5a\xa8\x26\xca\x73\x7d\x7d\x78\x53\x55\xd9\x64\x92\x84\x81\xf7\x44\x07\x66\x21\xd5\xf5\x84\x77\x21\xff\x2f\x5d\xb0\xfb\xaf\x5f\xb0\xfb\xfa\x82\x99\x97\xfe\xae\xc8\x42\xe0\x1d\x98\xcb\x0f\x83\xde\x27\x89\xde\x27\x74\x84\xee\xf5\xdd\x82\xc6\xed\x93\x9b\x9f\xa1\x82\xb4\x3f\x34\x8d\xaf\x3f\xdd\xa8\x25\x45\xff\xbf\x58\x66\xbb\xfc\x40\x2e\xf5\x88\xf6\x8e\x03\xd7\xc1\xfc\x27\x79\xc9\xc2\x64\x67\x56\x52\xb7\x3f\x92\x95\xfc\xa3\xcb\x26\xce\x48\xf6\x4a\xac\xe3\xdc\xfa\x40\x60\x53\x6f\x1e\x08\x9a\x38\x03\x59\xb3\x76\xc7\x6c\x6f\x19\x54\x39\x48\xfb\xde\x9d\xe8\x63\xce\xe6\x65\x59\x50\x4e\x12\xf5\xf6\x26\xdc\xdc\x35\x80\xac\xbe\xda\xce\xf2\x7f\x36\xca\x97\xb6\xa5\xfe\x6d\x19\xc7\x49\x6e\x0d\x7e\xe1\x2f\xde\x19\xa7\xea\x7c\x67\xe3\xb5\xac\x10\xc3\x23\x76\xbb\xb4\x93\x18\x2d\xcd\x3e\x2f\xab\x8e\x87\xe8\x90\xbc\xf8\x4b\xed\x85\x98\x70\x89\x7a\xb2\x3c\xe2\x85\x75\x70\x0a\x7e\x0b\x06\x6e\x16\x64\x1b\xca\xe1\x1a\x28\x87\x75\x28\xff\xb5\x01\xca\xe1\x5f\xfd\x50\x0e\xff\x5a\x87\x72\xba\x09\xca\x77\x6b\xa0\x7c\x57\x87\x72\xbe\x09\xca\x8b\x35\x50\x5e\xd4\xa1\x5c\x6d\x80\xf2\x83\x1f\xc8\x0f\x75\x18\x7f\xdb\x00\xe3\x7b\x3f\x8c\xef\xeb\x30\xde\x6d\x80\x51\x7f\x61\x5a\xc1\xf8\xb6\x0e\xe3\x6e\x3d\x8c\x1a\x84\xa5\xaf\x9d\xb3\x47\x6d\x6a\x78\x24\x90\xea\xae\xe3\xbd\x6e\x93\xf9\x96\x7e\xc4\x14\x9c\x35\xdc\xd7\x6d\xb2\xdf\x1f\x9b\xe0\xac\xe3\xbf\x6e\x93\x01\xf1\x46\x38\x6b\x38\xb0\xdb\x64\xc1\xf1\x46\x38\x6b\x78\xb0\xdb\x64\xc2\x72\x13\x9c\x1f\xaa\xed\xad\x06\xa8\xc1\x88\xf9\x26\x38\x6b\x38\xb1\xdb\x60\xc5\xff\xf9\x3f\xd6\x81\x39\x24\xdd\x35\xbc\xd8\x6d\x30\xe3\x6c\x3d\x2e\x3e\x1e\xdb\x92\x1b\xc3\xb2\x63\x9c\x54\x07\xd2\x9a\xd9\x14\x7e\xf2\xee\xe5\xaf\xb7\x97\xa7\x17\x67\xa7\x97\xb7\xef\x3f\xbe\x53\x5f\x44\xac\xde\x7a\x21\x8c\x61\x78\x57\x3c\x58\x9f\x95\xed\x35\xe1\xf1\xd4\x4a\xca\xe6\x9a\x76\xfb\x90\x72\x4d\xfa\xbc\x46\x73\xae\x6f\xd1\xd2\x7c\x82\x8a\x3c\x5b\xa2\x71\x4a\x19\x37\x7d\x6b\xe8\xec\xa3\x20\x0a\x4c\x34\xa1\x0b\xf8\xb8\xd6\xb8\x71\x92\xd3\x51\xf2\x6a\x12\x6e\x18\x8b\x82\xc5\xca\x2c\x8d\x49\x78\xd0\xa9\x03\xab\xc5\x00\xc9\xe6\x90\xa2\x44\x26\x4e\xaa\x12\x1b\xa9\x57\x73\xaa\xf4\x46\x7d\x74\x0d\xde\x20\x79\x92\xd6\x4f\xf5\xd4\x49\x7d\x99\xf5\x11\xee\x53\x43\x6f\x1e\xc9\x96\x4a\x10\xd9\x92\x37\xaa\x02\x4c\x6d\xdb\x3c\xc7\x8d\xf7\xf8\x3d\x2d\x6a\x79\xa1\xfc\x3b\x3d\x20\x5b\xe5\x57\x01\x2f\x2b\xe0\xf6\xf1\xe2\x6d\x75\xc7\x6e\xb7\xf2\xda\xee\x4e\x03\x79\x65\xb8\xaa\x82\x39\x9d\x5a\x7d\xef\x00\x43\xe1\x24\x91\x6e\x24\x64\xbe\xb5\xf5\x34\x0c\xbe\xc1\x49\x72\xab\xbe\xbf\xa2\x12\x7d\x3a\xad\xe5\x77\x6e\x44\x51\x07\x7d\x5e\xb5\x9b\x76\x45\x6d\xfa\x7a\x42\x4d\x12\x88\xc9\xa9\xf0\x4f\xb1\x16\x90\x0f\x9d\x11\x4c\xe5\xe7\xcf\x82\xa0\x26\x93\x3a\x08\x4a\x11\x0f\xd6\xf6\x5c\xbf\xf1\xe2\x87\x13\xb1\xf9\x48\xda\x8c\xe1\x61\x1b\xf8\x8e\x87\xad\x67\x2d\xf3\x2a\x5a\x05\xe3\x0d\xc9\x4a\xe3\x16\xac\x4f\xe6\xef\xb5\x66\xa1\x1d\xca\x51\x87\x21\x27\x5c\x75\x61\xa1\x85\xe9\x56\x6a\x69\x2a\xdb\xd4\xd2\xdf\x1d\x74\xf9\xa6\x89\xab\x74\x71\x00\xc9\x9e\x9a\x6f\xfe\x59\x5f\x96\x52\x0e\x7f\xf5\x45\x44\x69\x66\x8b\x95\x95\x0e\x92\x8f\x17\x6f\xab\xa5\x6d\x5b\xd5\xd2\xdc\xac\xad\x7d\x7b\x0f\x52\x25\xee\x39\xe9\xf2\xa4\xf0\x49\x16\xd4\xf1\x13\xc0\x54\x72\x75\xda\xca\xbd\xd8\x8c\xf0\xd5\x51\x21\xc6\xf9\x58\xe5\x95\x16\xe4\xea\xf5\xd0\xfb\x0f\x57\xa7\xfd\x5a\x76\xa5\x11\x41\x77\xa4\xe4\x90\x83\x6a\x99\xc7\x32\x42\xa0\x37\xe7\x69\x26\xd4\xa4\xfe\x1b\x17\xf9\x7d\x34\x29\xfa\x00\xf7\x6d\x9a\xdf\xbd\x2e\xe8\xa9\x89\xb4\xdb\xb0\x14\x86\x2c\x7e\xe1\x85\x55\x95\xbb\x0b\x44\x5d\xd6\xa9\xe0\x04\x9a\x4d\xa4\x9c\x41\x4a\x21\x3b\x38\xaf\xa6\x01\x24\x1d\xaa\xa4\x4b\x3a\x42\xe6\x4f\xf3\xaa\x05\xe2\xc3\xe8\x13\x89\xb9\xce\x3e\x67\x33\xee\x84\xe4\x84\x62\x2e\x79\x57\x36\x73\x94\x8f\xc6\xdf\xd1\xe9\x4f\x65\x04\x56\x68\xc1\xd6\x41\xd8\xf2\x1b\x83\x32\xf6\xf5\x99\xfa\x14\x92\x52\xaf\xc0\x23\x97\x1c\x73\x12\x7e\x5e\x75\x50\x10\x74\x90\x8c\xe7\xf9\x51\x9c\xe9\x2c\xd2\x6e\x15\x18\x8b\x3b\xed\x75\x92\xec\x07\x94\x6e\x2c\x8d\x77\xb9\x54\x32\xb7\x0a\x40\x1b\x7d\x56\x53\x9c\x80\x3f\x1b\xda\x79\xde\xd3\xf0\x52\x7d\x87\x0d\xa2\xde\xa5\xae\x32\xff\xee\xe8\x37\x03\xcd\x56\x26\x86\x17\xc1\x03\x4c\x12\xb7\x8b\xbc\xb4\x83\x69\x9d\xe5\xf7\x38\x4b\x13\x8f\x3e\x92\x49\xd7\x6c\x7d\x26\xbb\x09\xf3\x42\x2d\xfb\x6b\x5a\xcc\x3e\xc8\x01\x14\x80\xe6\x70\x1d\x74\xb0\x23\x65\xa2\x6a\x74\x79\xbb\x88\x86\xa8\xf7\xdf\x93\xdf\x93\xfd\xdf\xa3\x68\x7f\x18\xed\x3f\xed\x3d\x8e\x58\x9e\x19\xda\xf4\x02\xee\xbc\x9a\x97\x99\xbe\x8e\x57\xd3\xb4\xca\x1b\x6b\x5f\xd5\xd5\xb6\xa0\x47\x4f\x2e\xe2\x84\x71\x1b\xde\xc0\xff\xb2\xcf\xd6\x49\x6e\x5a\x8f\x35\xec\xd1\x91\x2c\x7b\x56\xe9\x1c\xb1\xe1\x5a\x0d\x2a\x63\xa2\x71\xc6\xaa\xed\xb5\x25\x7c\x92\xf7\xc3\x58\xe8\x5f\x80\xe7\x24\x55\x04\x68\xf2\xab\xbd\xa1\x35\xa4\xde\x64\xf3\xf9\x6c\x44\xe8\x87\xb1\x1c\xf4\x75\x41\x05\x14\x2d\xb0\x36\x3a\x3b\x2f\x43\x55\x21\x83\x53\xd9\x2f\x29\x9f\x86\x0d\x24\x15\xb1\xcd\x7b\x63\x8a\x02\x9b\xf0\xd9\x4e\x89\x6d\x93\xa8\x8c\xdb\xf5\xe3\xb4\xad\x97\xc6\x6b\xa0\x9a\x85\xee\x46\xb2\x13\x4d\x8c\xd1\xd3\x20\x89\xa2\x85\x61\xc3\x46\xce\xbe\xca\x06\xb5\xa4\xfb\xc3\xf8\x43\xae\xf6\xe5\x26\x7e\x66\x9d\x25\x90\x97\x71\x3c\x9f\xcd\x33\xcc\xe1\x65\xb1\x1d\x94\xc9\x1a\x8e\x45\xfb\x2a\x2b\x41\x03\xac\x89\xbd\xab\xbe\xe6\x5c\xcf\x54\x67\xb5\x7e\xb4\xa8\xad\x9f\xfc\x76\x35\xec\xa4\x3e\x44\x2e\x73\x37\xc2\x84\xec\x45\xac\x7a\xbf\xc7\x33\xf2\x32\x4f\xf4\xbb\x1e\x5c\xae\xa8\xb4\x5c\x87\x2d\x6b\x33\xaf\x9a\x9b\x0f\xd8\xdb\x7d\x21\x0b\x79\xad\xb1\x06\x9a\x90\xb8\x48\xc8\xc7\x8b\xb3\x57\xc5\xac\x2c\x72\x92\x6b\x5a\x3a\x00\x0e\x6f\xaa\x7c\xa6\xbf\xef\x43\x42\x53\x14\xb4\x75\x82\x72\x21\x49\x36\x0a\x43\x14\x70\x3c\xb2\x5e\xa9\x71\x87\x34\xb9\x2c\xac\x62\xf9\x7d\x02\x8e\x47\x28\x65\x10\xb3\x37\x21\x54\xf9\xa1\x6d\x4b\xf5\xba\x1a\xe6\xc6\x4c\xf5\x67\x9d\xf9\x70\xe5\x59\xfe\x66\xa2\xc2\x6d\x8b\x5e\xd7\x63\xf6\x52\x5b\x46\x9b\x1a\x25\x98\x08\x2b\x25\x55\x6c\x0a\xa7\xe5\x47\x8e\xe7\x31\xb5\x1a\xd6\x4b\xcd\xea\x32\x5c\x56\x6a\x0c\xfd\x1a\x38\x75\x94\xaf\x6b\xf2\x49\xb6\x94\x8f\xd1\x1d\x59\x32\x67\xa4\x76\x93\x49\xef\xaa\x4f\x4e\x5b\x90\xae\x15\x0a\xfb\xe8\x8e\x2c\x6f\xb4\xdd\xaa\xa0\x5c\x8b\xb2\x46\xc0\xbb\xd5\xdb\xd8\xf4\x57\x53\xc2\x08\xe2\x0f\x85\xca\x42\xc0\x50\x98\xb2\x13\x52\x52\x12\x63\x4e\xe4\x39\x48\x98\xdf\x38\x4f\x10\x25\x49\x4a\x49\xcc\xaf\x8a\x77\xe9\x44\x50\x2e\xf9\x78\xf1\xb6\x2d\xa0\x60\x4a\x10\x4e\x12\x92\x28\xd7\x46\x41\xe1\x0b\xae\x0f\x98\x26\xf0\xc6\x37\xe6\xe9\x28\xcd\x52\xbe\x14\x47\x86\x22\xd3\x59\xe0\xa5\xb7\x3b\xda\x33\x39\x69\x7d\x43\x6f\x38\xa8\x4e\x31\x9b\x6e\xd8\x40\xab\x0f\x5e\x68\x1d\x2b\x85\x2e\x79\x4d\xf1\x44\x65\x35\xf1\x88\xa1\x6f\x14\x79\x9b\x4b\x97\x46\xb2\x2c\x1f\x46\x0d\xa8\x52\xfd\xe1\x61\x5b\xca\x56\x42\x8b\x12\x2e\xf6\x05\x1c\xf4\x0d\x38\xbf\x62\x08\x13\x0a\x49\xc3\x85\x67\xa1\x5c\x19\x83\x54\x48\xd9\x6a\xaf\x22\x93\x77\x21\xac\x63\xd1\x9f\x9b\xa6\xe7\x4c\xf4\x67\x66\xeb\x97\x80\xba\x53\xc4\xd9\x60\x0b\x57\xea\x2a\xf5\x6c\xc4\xce\x23\xfd\xa2\x8d\x2d\x55\xc5\x2e\x02\xb5\x59\xa4\x8a\x9a\x34\x69\x79\x5a\xb9\xa2\x0d\x19\x3d\xfc\x27\xb0\x1a\x91\x3d\x59\x4a\x6a\x27\x2e\x58\xe8\xde\xf3\xe7\x7b\xe8\x39\x92\x79\x9a\x54\xd6\x0a\x34\x95\x87\x1a\x3d\x0b\x26\x5a\x3c\x7f\xde\x53\x6e\x39\x3b\xdd\x85\x72\xcc\x99\x64\xe6\x9e\x6c\xec\x8a\xb5\xb6\x7a\xe2\xe4\xf7\xd2\xba\x00\xbd\x5b\xb9\xe5\xf6\x74\x5e\xe9\x46\x4a\xe3\xbe\xf7\xf8\xa7\xb3\xae\xcb\x2f\x49\x9f\x66\xf2\xcd\x90\x24\xbd\x8f\x6a\x90\x07\x56\x63\xf5\xb1\xb2\xa7\xa1\xee\xd5\xae\xde\x6d\x69\xa5\xba\xad\xae\x8c\x8a\xf1\x38\x6c\x81\xaf\xac\x65\xef\x8f\xeb\xf2\xbb\x5b\x97\xee\x42\x85\xcb\x50\xf0\x0f\x79\x35\xd8\xc0\xde\x06\xab\x51\x72\x3d\x48\xc7\x97\xe3\x1e\xde\xf1\x8c\xa5\x34\xc9\x28\x93\x96\xe7\xe3\x64\x2d\x27\x75\xa6\x83\xe1\x3a\x3c\x1c\x97\xf9\xda\x11\x00\xfe\x76\xe8\xe3\xb1\x07\xbc\xcd\xdf\xb0\xb6\x4e\xd2\x94\xcd\xeb\xaa\xc5\x40\xfa\x4e\xdc\x04\xc7\x66\x39\xd9\x65\x3a\x2b\xf5\xe7\x32\xdc\xf3\xbf\xf5\x7d\x05\x90\xe9\x0f\xe3\x50\x65\x96\x17\xa7\xfe\xee\x61\x15\xbe\xec\x42\xa9\xeb\x4f\x67\xcd\x32\xc2\xd5\xbb\xb3\x7f\x77\x85\xe3\x11\xfe\x67\x7b\x02\x53\xcc\x44\x4b\x05\x4e\xdb\x53\x0e\xf0\x6d\xd8\xbb\x20\x2a\xec\x3d\x90\xd4\x19\x66\xeb\x18\x3a\x71\x98\x99\xb6\xa7\x07\x38\x03\x65\xf3\x41\x7d\xd5\x5e\x15\x73\xd8\x21\x3c\xdd\xec\x5c\x1f\x5e\x42\xfa\xd0\x06\xac\x2d\xc8\x5d\xf4\xdd\x41\xc7\x1a\x4a\xcb\xa6\xf7\xd3\x50\x86\xf2\x8d\x4f\x43\x35\x87\xd2\x8b\xb3\xe1\x2b\x0d\x5e\x14\x07\x8f\x55\x0d\xd2\x07\x69\xbe\x2a\x21\x61\x5f\x12\xe7\xe3\x0f\x8f\xfb\x3e\x84\x1d\x88\x0b\xa2\xa6\xa5\xde\x92\x32\x23\xa0\x0a\x99\x7f\x1d\x2a\xca\x59\xbd\x46\xcb\x3a\x21\x73\x7e\x65\x56\x7d\x17\xb0\xa9\x8a\x6a\x4a\xba\x6a\x99\xb2\xae\x01\x30\xd8\xc0\x11\xde\x1d\xa8\x23\xdf\xec\x19\xd4\x09\x38\x1e\xff\x33\x28\x68\x91\xe7\x6b\xa9\xa3\x66\xef\xa5\x8c\x45\xb9\x1a\x75\x1c\x60\x7f\x92\x40\x60\x49\x5a\x14\x5a\x37\x71\x8b\x60\x33\x5d\x56\x25\xa8\x6b\x78\x68\xeb\xb6\x94\xb3\x19\x58\x77\x65\x72\x13\x90\x29\x45\xcd\x47\x13\xdc\x9d\x69\x53\xbb\xc8\x7c\x9f\xc5\x20\xe5\xdd\xa8\xf4\x31\xe6\x84\x8c\x09\x15\x07\xd8\x7b\x6d\x69\x15\x63\x34\xce\xe1\x24\xf2\x80\x53\x7e\x4e\x68\x5a\x24\x02\x3d\xb9\x51\x90\xea\x03\x0e\xe2\xe0\x1b\xe3\x2c\x23\xe2\x80\x53\x12\x61\x4d\x67\xcb\xca\xd6\x4e\xc8\xa8\x98\xe7\x31\x09\xc7\x79\xc7\x02\x65\x25\xcf\x90\x29\x53\xac\x73\x56\xc3\x2e\x70\x12\xbe\xa8\xf6\x96\x17\x01\x53\x48\xa4\x81\xe9\x64\x2e\x4c\x6a\xeb\x93\x72\x2c\x2e\x4a\xe2\x7c\x68\xae\xc2\x3a\xbf\x2f\xee\xc8\x6b\xcf\xb7\x89\xc7\x79\x84\xcb\x32\x5b\x86\xd0\xbb\x03\xe0\x9d\xe0\x32\x85\x01\x04\xe2\x99\x8c\x2f\x1a\x9e\x33\xc9\x81\xcc\xae\x68\x9f\x3d\xec\x94\x02\xf2\x76\xc0\x50\x82\xe4\x3c\xe5\xcb\x77\xf2\x1b\x2e\x30\x52\xf0\x2c\xe8\xa3\xe0\x19\x9e\x95\x03\xfd\x49\x88\x23\x28\xc9\xb8\x29\x38\x86\x82\x89\x29\x68\x05\xad\x3e\x6a\x3d\xfb\xc7\xbc\xe0\x03\xf5\x81\xfa\xa0\x15\x88\xa2\x6f\xbe\xfd\xc1\x94\xf4\x64\xc9\xe2\xc5\xeb\x41\xcb\xa4\x80\x54\x04\x50\xe1\x06\x0a\xbd\xca\x81\x72\xfd\xec\xe8\x38\x68\xfd\xde\xbb\xe9\x4d\x3a\xd6\x77\x50\x58\xcd\x28\x36\xd3\xb8\x66\xe6\x0c\x6d\x53\x20\xcd\x53\xae\xa8\x2e\x13\xf9\x5d\x12\x3e\x2f\x55\x08\x7d\x8c\xe3\x29\x51\xdf\xef\xa9\x3c\x23\x4e\xc2\x3f\xef\xf7\xbd\x18\xc7\x3c\x8d\x7b\x9f\x98\x3c\x10\xdc\x72\x32\x2b\x33\xcc\x75\x2c\xf4\x08\xd3\x1f\xef\x87\xe2\x94\xf0\xd3\xc7\xb3\xb7\x27\xb7\x3f\x9f\x5e\x5c\x9e\x7d\x78\xaf\xde\xa6\x6c\xa4\xf5\x13\xf2\x24\x30\xdc\xb3\x24\xed\x4a\x41\x54\x21\x9a\x5a\x9c\xde\xcd\x19\x17\x38\xeb\xe3\x9d\xe8\x39\xb0\xad\x59\xef\x91\xdc\xcd\xd7\xe0\x3d\x8d\xd6\xec\x56\x3b\x14\x93\x22\x75\x81\xef\xb9\x3c\x0b\xad\xac\x00\xe0\xc1\x91\xe4\xae\x49\x7f\xb5\x2c\x4f\x43\xd1\xa0\x3d\xd8\xfb\x5f\x01\x00\x00\xff\xff\x8a\x26\x0c\xa4\x3b\x93\x00\x00") +var _pkgUiStaticJsGraphJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x7d\xeb\x7a\xdb\x38\xb2\xe0\xef\xf5\x53\x20\x9c\x7c\x11\x15\x4b\x94\x9d\x3e\xdd\x3b\x23\x59\xee\x4d\xc7\xce\xc4\xe7\xe4\x36\xb6\xd3\x97\xe3\xf6\xf1\x07\x91\x90\xc4\x98\x22\x39\x00\x64\x5b\x9d\xe8\xb1\xf6\x05\xf6\xc9\xf6\x43\xe1\x42\x80\x04\x25\xb9\x7b\x66\xbe\xbd\xe4\x87\x1c\x82\x40\xa1\x50\xa8\x2a\x14\x0a\x85\xe2\x1d\xa6\xe8\x23\x2d\x16\x84\xcf\xc9\x92\xa1\xb1\xfd\xf0\xf5\x2b\xfa\xb2\x1e\xed\x89\x2a\x33\x8a\xcb\xf9\x25\x59\x94\x19\xe6\x64\xb4\x07\x65\x67\xef\x3f\x7e\xba\xbc\x39\x39\xfd\xe1\xc3\xa7\xf7\xaf\x4e\x6f\x7e\x7a\x79\x76\x89\xc6\xe8\xdb\x83\x83\x11\x1a\x0c\xd0\x82\x41\xa5\x8b\xd3\x57\x1f\xde\x9f\xa0\x31\x3a\x3c\x38\x38\x18\xed\xed\x55\xe0\xa3\xbf\x0a\x98\x68\x8c\xa6\xcb\x3c\xe6\x69\x91\x87\x24\x23\x0b\x92\xf3\x1e\x2a\x4a\xf1\xcc\x7a\x68\x8e\xf3\x24\x23\xaf\xe6\x38\x9f\x11\xfd\x74\x4e\x16\xc5\x1d\xe9\xa2\x2f\x7b\x08\xf1\x79\xca\x22\x92\xa1\x31\x52\x6d\x47\xba\x10\x10\x7e\x73\xf9\xee\x2d\x1a\xa3\x7c\x99\x65\xe6\x85\x82\x8d\xc6\xba\x17\xf3\xc6\xee\x0c\x8d\x9d\xbe\x6b\x75\x24\x0a\x36\xea\x12\x1d\xe4\xa0\x18\x8a\x16\x5d\xd1\x74\x6d\xda\xd3\x34\xbe\x65\x73\x7c\xaf\xc7\xee\xa0\x96\x60\x8e\xd1\x18\x5d\x5d\x8f\xf6\x74\x51\x9a\xa7\x3c\xc5\x59\xfa\x1b\x09\xbb\xa3\xbd\xb5\x87\x80\x11\x4f\x17\xe4\x35\x8e\x79\x41\xc5\xa0\x04\x1a\xc1\x2a\x18\xa2\xef\x0e\xd0\x73\xf9\xf3\xe2\xdf\xd0\x73\xf4\xcd\x77\xdf\xf6\xc4\xab\xfb\xe6\xab\xff\x0e\x2f\x92\xda\x0b\x28\x9c\x57\x85\xf0\xbc\x80\x67\xf8\x2f\x0b\x86\xe8\xd0\x8f\x11\xe3\xa4\xfc\x11\x67\x4b\x22\x10\xba\x12\x95\x0f\x59\xd0\x43\xc1\xe1\x81\xfc\xb3\x10\xbf\xdf\xc2\xef\xa1\xfc\xf3\xcd\x81\x7c\x9a\x8b\xdf\x17\xf0\xfb\x1d\xfc\x1e\xca\x87\xc3\x04\x5e\x24\x01\x74\x7d\x78\x0f\x4f\xf0\xfb\x6f\xf0\xfb\x67\xf8\x3d\x5c\x41\xf9\x2a\xd8\xbb\xf6\xa1\x95\x2f\x17\xf0\x1f\x81\x95\x8f\x15\xa3\x92\x16\xbc\xe0\xab\x92\x58\x64\x6f\x4e\xb2\xe0\x6a\x46\xb2\x29\x1a\xc3\x14\x89\xd9\x13\x8f\x51\x9a\x38\xd2\x53\xef\x74\x7f\x1f\x66\x75\x30\x40\x17\x84\xa3\x84\x4c\xf1\x32\xe3\x9a\x07\x23\x0d\x44\x3f\x03\x30\x05\x76\x54\x7f\x49\x05\x4b\xde\xa4\x79\xb9\xe4\xba\x96\xef\xd5\xd7\xaf\x40\x51\xd1\x3c\x9d\xa2\xd0\xa9\xc7\xf1\x04\x8d\xc7\x63\xb4\xcc\x13\x32\x4d\x73\x92\x68\x06\x6e\xd6\x42\x87\xc0\xc2\x3e\x28\x0b\xfc\x70\xc3\x8a\x25\x8d\xc9\x0d\x25\xac\xc8\x96\xa2\xb8\x09\xf7\xbf\xd5\xc0\xb6\x34\x43\xc1\x01\x0b\x64\x5f\x92\x50\x27\x14\xdf\x4b\xcd\x83\xe2\x22\xe7\xb4\xc8\x18\xc2\x79\x02\x0f\x38\xcd\x09\x45\x53\x5a\x2c\xd0\x1b\x90\xb9\x09\xa6\x0c\x71\xa5\xa1\xa2\x3d\x35\x51\x95\xb4\xcb\xe1\x75\x4a\xcc\xe7\x1f\x29\x99\xa6\x0f\x9d\x21\xfa\xf8\xf2\xf2\xcd\xcd\xc7\xf3\xd3\xd7\x67\x3f\xf7\xe4\xeb\xc9\x32\xcd\x92\x1f\x09\x65\x69\x91\x77\x86\xe8\x87\x4f\x67\x6f\x4f\x6e\x7e\x3c\x3d\xbf\x38\xfb\xf0\x5e\x0b\xf2\xe7\xbf\x2d\x09\x5d\x45\xe4\x81\x93\x3c\x09\x8d\xae\xb2\x87\xd8\x35\x73\x66\xeb\xa1\xa7\xe1\xbb\x25\xe3\x38\x9e\x93\x88\x92\x3c\x21\x34\x74\xd4\xaa\xd1\x7b\xdd\xaa\x39\xc9\x22\x5c\x96\xa2\x1f\x17\x5a\x57\x33\xd3\x5f\x09\x47\x94\x4c\x09\x25\x79\x4c\x18\xe2\x05\xc2\x59\x86\xf8\x9c\xa0\x34\xe7\x84\x12\xc6\xd3\x7c\xa6\xb5\x23\x43\x69\x0e\xef\x2a\xa2\x4a\x3a\xe2\x3c\x91\xe0\x26\x69\x9e\x20\x72\x47\x72\xae\x54\x19\x05\xde\x34\x4b\xc0\x4f\x54\xa0\x43\x35\xdb\x91\x2c\x9a\xa6\x79\x12\x06\x7f\x82\xb7\x37\xf7\xf2\x75\x80\xf6\x35\xf3\x56\x43\xf9\xbb\xa0\xda\xeb\x82\x2e\xd0\xd8\x81\xa5\x20\xc8\xf7\x37\xd3\x82\x2e\x02\x33\xba\x97\x4b\x5e\xf4\x29\x61\x42\x10\x05\xde\x9c\x3c\x70\x84\x29\xc1\xa8\xc8\x91\xe4\xf2\x82\xa2\x45\xb1\x64\x24\xce\xd2\xf8\x56\xa1\x2a\x5b\x5c\x92\x07\x0e\x75\x9d\x25\x46\x33\x3a\x70\xc7\x74\xca\x08\x87\xd5\x23\x92\xff\x7f\x43\xd2\xd9\x9c\xa3\xbe\x28\x89\xb3\x94\xe4\xaa\x64\x04\x6d\x9e\x8a\xf6\x51\xcc\x58\xd8\x99\x43\x71\xa7\x87\x3a\x78\xc9\x8b\x4e\xbd\x94\x64\x11\x8b\x69\x91\x65\x0a\xe0\xbe\xea\x4b\x2f\x07\x66\x7e\x1f\x4a\xea\xa7\x07\x57\xd8\x5f\xe5\x78\x41\xc6\xa2\xde\x75\x60\xf1\xc5\x43\x49\xa3\x5b\xb2\x2a\x29\x61\x2c\xac\x86\xa7\x47\x17\x17\x39\xe3\x88\x08\x16\x10\x12\xfc\x8d\xc4\x5f\x08\x30\x89\xee\xe7\x69\x3c\x47\xe3\xb1\x7a\xfd\xec\x19\x7a\x42\x22\x36\x4f\xa7\xfc\x3f\xc8\x4a\x03\xa8\x4f\x5a\xc4\x96\x93\x45\xca\xc3\xee\x48\xbd\x26\x51\x49\x81\x51\x4e\xa4\x2a\xd3\x6f\xd6\x8a\x52\xb0\xf8\x45\x45\x1e\x76\x6e\xc9\x6a\x59\xca\xd9\xea\xf4\x50\x42\x26\xc5\x32\x8f\x49\xd8\x58\x3b\x51\x6d\xde\xaa\xf5\x13\xa1\x75\xcf\x67\x6e\x48\x41\x59\x77\x5d\x7a\x46\xc0\x0a\x1e\xa2\xb4\x81\x6f\x02\x80\x45\x5f\x8a\x9c\x6d\x06\x58\xf5\x12\x92\x2c\xcb\x1f\x78\xae\x25\xa1\x22\x94\x62\x67\xa8\x70\x33\xe1\xb9\x3d\x6b\x39\x9e\x64\xe4\x44\xbc\x69\x6b\x07\x64\x92\x73\x0e\x10\xec\x49\x2f\x31\x15\xab\xd2\x39\x61\x65\x91\x33\xb2\xa9\x77\x55\x55\xe8\x57\xa8\x5b\x43\xa4\x06\x69\x07\x64\xea\x00\x6d\xbc\x60\xd1\x39\xb3\x97\xa3\x0d\x80\xac\x15\xca\x86\x21\xf4\xe2\x2d\x49\x36\x8d\x49\x55\xa9\x0d\x45\x95\xee\xd0\xb3\xaa\x69\xf7\x9a\xe6\x8c\x50\xfe\x8e\x70\x9a\xc6\x6d\x10\x18\xc9\x48\xac\x40\xc8\xfa\x37\x0b\x68\xe0\x90\x80\x4c\x29\x61\xf3\x33\x21\x51\x77\x38\xdb\x05\x96\x6a\x62\x43\x59\xe0\x87\x0b\x58\x17\xcf\xcd\xb2\xb8\x91\xac\x36\x38\xef\xa2\x6a\xd1\xd9\x70\xae\x50\x0d\x45\x46\x2e\x61\x6d\xf7\x29\x62\x55\x21\xa8\x2d\x62\xa2\x01\x6a\x69\x22\xb5\xbf\x59\x4f\x02\x4b\x50\x38\x9e\x30\x7f\x2b\x7c\x25\x0c\xde\x3e\x2f\x66\xb3\x8c\x8c\x3b\x1c\x4f\x3a\x36\x31\x44\xc3\x88\xfc\xbd\x61\xb7\x74\xc5\x4f\x18\xb0\x79\x71\x5f\xaf\x5d\xe4\xb2\x3c\x8f\x26\x50\x35\xe8\xa1\xa6\x16\x10\x8a\x9f\x63\x3a\x03\xc5\xff\x34\x24\x91\x7c\x50\x8a\xc6\x63\xff\xc8\xf7\x42\x66\x48\xce\xc3\x6e\x94\xe6\x09\x79\x08\xed\xfa\xb6\x8e\xd0\x2f\x84\xae\x7d\x1a\x06\x7f\x12\x6b\xa1\x82\x80\x39\xa7\x61\x80\x69\x8a\xfb\xda\x9e\x09\xba\xdd\x68\x8e\xd9\xab\x0c\x33\x16\x06\x94\x64\x05\x4e\x82\x6e\x4d\x09\x4b\xd5\x0b\x56\x87\xad\x65\xd7\x66\x99\x3c\x27\x7c\x49\x73\x24\x36\x1d\x0c\x4d\x8b\x78\xc9\xd0\x04\xc7\xb7\xc2\x1a\x80\x05\x26\xcd\x19\x27\x38\x41\xc5\x14\x49\x58\xc2\x28\x88\x7c\x42\x10\x4d\x60\x6a\x6e\xc9\x2a\x29\xee\x73\x61\x4e\x53\x80\xed\xa5\x64\xa5\x30\xa1\x4f\x87\x24\x50\x7c\x87\xb3\xd0\x7d\xea\xaa\x3a\x12\x6a\xcb\x22\xb2\xee\x56\xea\x98\xd2\xa2\x65\x81\x94\xef\x82\x6e\x34\x4f\x13\x45\x75\x68\x72\x8f\x69\x2e\x6c\x1e\x7f\x23\xf5\xb6\xd9\x0c\x2a\xbf\x94\xe6\x42\x3b\x8b\x8b\x85\xa3\x2e\x18\x5a\x3a\x0d\x04\xa7\x89\x55\x7b\xf5\xf2\x21\x65\xad\xb5\x57\x37\xf8\x21\x65\x56\xf5\x8c\xcc\x48\x9e\xb4\xa0\x23\x5f\xda\x7a\xb0\x4c\xf3\x9c\xb4\xd1\x4a\xbd\xb5\xd7\xa2\x3b\x9c\x5d\x70\xcc\x5b\x84\x13\xde\xdf\x30\x51\xc1\x96\x66\x92\x27\x27\x98\x13\x7f\x1b\x4b\xd7\x92\x3c\x69\xea\x78\xd5\x58\xec\x73\x89\xd8\xb5\x96\x69\x7c\x4b\x68\x28\x99\x29\x2b\x62\x9c\x91\x21\xea\x90\xbc\x23\x8d\x71\x61\x0a\x62\x3e\x44\x9d\x5f\x7e\xf9\xe5\x97\xfe\xbb\x77\xfd\x93\x13\xf4\xe6\xcd\x70\xb1\x50\xef\x79\x51\x64\x13\x4c\x3f\x66\x38\x06\xeb\x76\x88\x3a\x93\x82\xf3\x42\xbf\x67\x69\x42\x7e\x58\x5d\xa4\x09\x19\x22\x4e\x97\x44\x95\xce\x8b\xfb\xcb\x22\xc1\xab\x1f\x96\x9c\x17\x79\xfd\xd5\xab\x8c\x60\xda\x2c\x2c\x98\x03\x44\x60\xff\x9f\x45\x2e\xd0\xfd\x74\xf9\x0a\xfa\x5b\x77\xbd\x1b\x2d\x43\x08\x57\x68\x2a\x4a\xe0\xb0\x23\xfe\x7b\x99\x2e\xc8\x47\xa0\x47\xa7\x0b\x04\x6a\x03\xa3\x37\x63\x0e\x1c\xa1\xf8\x92\x52\xd9\x2d\xb6\xac\x76\xd1\x17\x9f\x0e\x51\xd8\xfa\x96\x2e\x6d\xfc\x34\x41\x2c\x4b\x81\xd7\xb9\xac\xae\x81\x18\x25\xc2\x2e\xcc\x42\xdc\xb0\xec\x94\xb4\xdb\xeb\xb5\xd4\x06\xb0\x57\xec\x1c\x76\x6a\x56\xf1\xa2\x10\xf3\xb9\x95\xc9\x64\xb5\x26\x9f\xc9\xf2\x3f\xcc\x66\x43\xc6\xfe\x6f\xe2\x34\x51\x93\x71\xbc\x28\xed\x85\x2e\x91\xc2\x9a\x93\x7b\x74\xd2\x60\x2a\xd3\xe2\xf9\xe1\xc1\xc1\x41\xb7\x62\xcf\x8a\x80\xad\xdc\x29\x7e\x24\x2f\x22\x92\x31\xe2\xf1\x0d\x58\x93\xe3\xf0\xfe\x0e\xc0\xdb\x01\x39\xdc\xaf\x20\xfd\x2e\xe6\xd7\x8e\x1c\xbe\xca\x08\x70\xae\xb4\x3c\x1b\xac\x2b\x2a\xa5\x71\x61\xac\xd2\xca\x4e\x95\xfc\xd8\x89\x66\xd9\xaa\x9c\x8b\x2a\x1d\x6b\xe5\x77\x65\x22\x6c\xac\xe8\x15\x14\x9c\x24\x6a\xf5\x9f\xf0\xbc\x5f\xd2\x74\x81\xe9\x2a\x30\xdb\x2c\x01\xd8\xaa\x63\x3a\xeb\xc7\x73\x12\xdf\xd6\xea\x51\xf0\x3b\x36\xaa\x2e\x73\xa8\x4c\x12\x5d\x5d\xcd\x59\x1b\x4a\x0e\x98\xc7\x61\xd5\xe8\x6a\x33\x66\xce\x20\xd6\xda\xc1\xe2\x4c\x4a\x68\x29\x19\x0b\xc7\xda\x1e\x4f\xd3\xd7\x47\x7b\xb1\xc3\xad\x96\xdc\x7f\xbf\xf8\xf0\xbe\x9a\x8d\xc1\x00\x9d\x4d\x2d\x97\xc8\x3d\x66\x48\xf5\xd2\x83\xe2\x82\xa6\xb3\x34\xc7\x19\x62\x84\xa6\x84\x21\xf0\xd1\xce\x0a\x8e\x16\x4b\x8e\x39\x49\x2a\x38\x21\x13\x9a\x25\xe9\x82\x8b\xea\x9e\xa0\x9c\x90\x44\x58\x60\x94\xc0\x66\x9c\x2e\x63\x8e\x52\x2e\x5d\x56\x0e\x64\x81\x11\xc0\x8d\xec\xf9\x50\xce\x60\x69\xdc\x52\x9c\x33\xa1\xa7\x4e\x84\xd0\xd4\xc6\x62\x6f\xbb\x1b\x1a\xb6\x41\x8b\xef\x51\xe7\xa0\x83\x86\x42\xe9\x6a\x73\xad\x4e\x6d\x03\x48\x2a\x7c\x70\x5f\x86\xf6\x46\x79\x30\x40\xb0\x87\xcd\xd2\x18\x0b\xea\x57\x96\x24\x83\xf2\x53\xd8\xe6\x7a\x57\x02\x77\x2d\xb0\xf6\xc3\xde\xf5\xc0\x92\xd1\x93\x6a\xbf\xed\x01\x5a\x97\x52\xbd\x3b\x6f\x95\x51\x87\x53\x6c\xa4\x6d\x51\x7d\x8c\xfc\x3d\x5a\x06\x1b\x52\xf8\x58\xb9\x7a\x94\x6c\xd5\xa5\x4b\x93\x33\xf4\xb8\x34\xda\x64\xcb\x3b\x65\x5e\x12\xb6\x71\x99\xdd\xab\xcd\x4d\x1f\xa5\x73\x01\x69\xe7\x82\xc5\x50\x1f\x5d\x4f\xc5\x06\xd6\xb2\x19\xab\xe6\xdf\xd8\xc6\x5c\x1f\x7d\x8e\x95\xad\x4b\x41\xd3\x1d\xb3\xcb\x92\xe0\x1f\x90\xcd\x76\xff\x0a\x95\xff\x2f\x50\xe0\x4d\xa2\xda\xcc\xe6\x21\xde\x26\xb6\xf3\x4d\xe8\x16\x82\xb6\x31\xa1\x1f\x2f\xd7\x0b\xd8\xe6\x88\xa9\xfa\xdd\x78\x96\x61\x41\x6a\x38\x50\xfd\xe3\xab\x9c\x32\xd6\xac\x68\x77\x80\xa5\x92\xb5\x27\x66\x73\x2d\x9f\x3f\xa2\xcd\x93\xa0\xc4\x66\x8a\x33\x46\x46\xc6\xba\xb4\xb7\x94\x66\xa7\xdc\x1c\x93\x34\xc9\x27\x60\xdf\x6a\x0f\x59\x7c\x03\x2e\xbe\xeb\xa0\xeb\x99\x51\xed\x71\x88\x29\xc1\x8c\x9c\x2b\x04\xed\x4e\x37\x01\x4f\xc8\x0e\xc0\x13\xe2\x01\xbe\x2b\xea\x24\x4f\x76\x41\xfc\x34\x4f\x1e\x89\xf6\x16\xc0\x1a\x69\x0b\xf0\xae\x28\x4b\x2b\x78\x17\xac\xdf\x41\xcd\x47\x22\xbe\x1d\xbc\xc6\xdd\x05\xef\xf5\x2e\x79\xf6\x96\x35\x97\x91\x74\x69\x8a\x77\x01\x25\xa5\xd8\x70\x05\x3d\xf4\x85\x93\x07\x3e\xf4\xc0\x03\xbd\xde\x43\x8b\x42\xec\xbc\x82\x09\x99\x16\x94\x04\xeb\x86\x1f\x4a\xbb\xa7\xc4\x5a\x43\x09\x3c\xa5\xf9\xac\x92\x79\x79\x66\x24\x14\x9c\x54\xfd\x9e\x6d\xa7\xf6\xd9\x8a\x4a\x6a\xaf\x69\x5a\x6c\xd4\x5c\xb2\xd6\x26\x69\x33\x8e\x59\xa1\x4b\xc5\x66\xe8\x84\xa6\x53\xe5\x0a\x1b\x0c\x90\x75\x16\x0d\x73\x85\xe6\x29\xe3\x05\x5d\xa9\x3d\xe0\x13\xd8\xd1\x5e\xf0\x82\xe2\x19\x89\x66\x84\x9f\x71\xb2\x08\x03\x55\xa9\xf2\x25\x3a\xd5\x58\xbd\x5a\x0f\xac\xd0\x88\x71\x9a\xe6\xb3\x74\xba\x0a\xaf\xae\xbb\xee\x66\xab\x2c\xca\x65\x86\x39\x39\x03\xfa\x0b\x1d\x2b\xe7\x80\x29\xcd\x60\x96\x38\xcb\xd7\x67\xd3\xa1\xa1\x7a\xd6\xfe\xe0\x81\xea\x10\xde\xa5\x47\xdb\x8a\xcc\xdc\xa3\x78\x59\x38\xa1\xc5\x3d\x23\x54\x34\xb6\x77\xbf\x5d\x41\x1f\x51\x18\x76\xd1\x40\x45\xa4\x88\x26\x4f\x23\xfc\x19\x3f\x84\x95\x25\x26\x50\x2a\x92\x21\x0a\xfe\x7a\x7a\x19\xf4\x4c\xf1\x92\x66\xce\xa1\x31\xda\x47\xc1\x00\x97\xe9\xe0\xee\x70\x00\x73\xf3\x3d\xfc\x8e\x39\x74\x61\x35\x14\xc6\xfc\xe5\xaa\x14\x4c\xfa\x99\x15\xb9\xf5\x06\xe8\xb3\x8c\x63\xc2\xd8\xb0\x1a\xa0\xa8\xd4\x83\xd3\xce\x0b\x8e\xf9\x92\xb9\x36\xa9\x24\xb6\xa8\x23\x6c\x7d\xbe\x64\xe8\xc9\x78\x8c\x02\x05\x26\xa8\x57\xae\xa6\x60\x5e\xdc\x9f\x52\x5a\xd0\x30\x80\x3f\x92\x9f\xd2\x7c\x06\x5e\x86\xc8\x35\x2d\xe5\x3f\xc9\xaf\x6e\xf9\xda\x79\x92\x73\x40\xef\x0c\xb5\x01\x2f\xd8\xce\x50\xc2\x96\x19\xbf\x3a\xb8\x1e\x35\x5a\x24\xe9\x54\xcc\xda\x3b\xcc\xe7\x11\x9e\xb0\xd0\x9e\xb0\xbe\x05\x4f\xf2\x96\x3b\x70\x68\x7b\x3c\x46\xdf\x1c\x34\x47\x0a\x41\x33\x62\x9c\x3f\x49\x3f\x6f\xd8\x18\x11\x42\xc1\x51\x92\xde\xa1\x58\xac\x9e\xe3\x5f\x03\x9c\x11\xca\x11\xfc\xf6\x95\x73\xf8\xd7\xe0\xf8\x88\x71\x5a\xe4\xb3\x63\x05\xe6\xc9\xd1\x40\x15\xa0\x13\xc2\x49\xcc\x49\x82\x02\xb4\xef\x01\x2e\x90\x8b\x78\xf1\x3a\x7d\x20\x49\xf8\xa2\xeb\xad\x13\x20\x26\xf6\x84\x09\x03\xba\x43\x13\x79\x7e\x8f\x26\x84\xdf\x13\x92\xa3\x55\xb1\x34\x4c\x0c\xfb\x49\xb1\x61\x94\x54\x89\xec\x08\x2d\x4a\x32\xb1\x29\x2d\x72\x84\xe3\x78\x49\x31\x27\x12\x24\x34\x01\xd8\x20\x3a\x0b\x38\x83\x8e\xf1\x92\x11\xb4\xcc\xc9\x43\x29\x47\x20\xd5\x89\x9c\x25\x16\x1d\x0d\x92\xf4\xee\x38\xa8\xe1\xdb\x6d\x9b\xfb\x75\xc5\xc3\xe0\x88\x1f\xfa\xf6\x65\xf2\x9f\x9f\xf9\x84\xd5\xe2\xe5\x3d\xd9\xc7\xba\x2d\xde\xa9\x52\x10\xad\x2a\x69\xa7\xa0\x9d\x9a\xd0\x7b\x45\x7e\x93\xc0\x67\x78\x42\xb2\xc1\xcd\x8d\x58\x18\x6e\x6e\x06\x77\x10\xf0\x64\x5a\xb6\x49\xfc\xe3\x64\xfd\x11\x72\xbe\x99\xc8\xf8\x0e\xa7\x99\xa0\x10\x92\x67\x97\xec\x89\x2b\xed\x75\x39\x5f\x57\x62\x57\xe2\x19\x79\x55\xe4\xd3\x74\x16\xe1\x2c\xab\x28\x6c\xe4\x1c\x96\x55\x5e\x24\xc5\x10\x25\x85\xf1\x7c\x00\x3e\x55\x83\xef\xd1\x07\x8a\x62\x9c\xa3\x94\xa3\xcf\x4b\xc6\x51\x96\xde\x11\xc1\xb8\x82\xb3\x45\x17\xa6\xbf\x69\x41\x51\x08\x7b\x2d\x88\xd3\x42\x29\x3a\xf2\xe3\x10\x65\x24\x9f\xf1\xf9\x08\xa5\xfb\xfb\x1e\x5a\xd8\x86\xc2\xd5\xc1\xb5\xb1\xd8\x71\x92\x84\x62\x45\xf8\x00\xcf\xa1\x17\xf4\x55\x7a\xdd\xf3\x77\x7a\x95\x5e\x77\xbb\x5e\x3a\x41\xa7\xd3\xe5\x6f\xbf\xad\xce\x41\xa2\x4c\xcc\x91\xfc\x07\xc2\x36\x84\x80\xbf\x9e\x43\x78\x51\xb7\x59\xbe\xc0\xe5\x10\x7d\x59\xb7\x76\x24\xac\x02\xc1\x5f\x78\x4e\xb0\x0c\x0e\xaa\x76\xfa\x1a\xce\x26\xb9\xfc\xfd\xec\xb2\xd6\x3e\xe8\x2d\xd2\xe9\x60\x68\x4b\x24\x20\x0b\xa8\xc8\x28\x15\xb9\x7d\x42\x63\x49\xa2\x37\xd2\x22\x89\x52\x66\xef\xe4\xac\xb9\x30\xb5\x34\x1b\xc4\x45\x1e\x63\xee\x9f\xc8\x2e\x1a\xfa\xe7\xd1\x8d\xa4\xe1\x86\x92\x92\x42\x78\xc9\x8b\x0b\xb0\x44\x87\xd2\x56\x53\x1e\x7a\xc0\x74\xa8\xfe\xca\xb2\x94\x93\x05\x1b\x82\x31\x21\x0b\x16\x98\xc7\x73\x62\xd1\x1d\x85\xa2\x4e\xdd\xe7\x78\x4f\xd0\x1c\xdf\x11\xc5\x00\xc0\xf5\xf1\x92\x52\x92\x73\x49\x87\x1e\x62\xb7\x69\x59\x77\x56\x59\xfc\x25\x09\x01\x2a\x01\x56\x3d\x78\x6c\x4c\x71\xb3\x81\x5d\x7d\xd4\x5e\x79\x81\x4b\xc1\xc1\xeb\x0d\x55\xa8\xe6\x73\x28\x8c\xa6\x69\xc6\x09\x0d\x2b\xe8\x91\xb2\xe0\xc3\x01\x1a\xcc\x7a\x28\x08\xba\x3d\xb5\x40\x4b\xfa\x39\xf2\x51\x52\xa1\x2b\xf5\xba\xeb\x58\x48\x65\xc1\xb8\x78\xa7\xd7\xe0\x6a\x8d\x5a\x77\xb7\xa2\x17\x4d\x0b\x7a\x8a\xe3\x79\x65\x9e\x53\x8f\xb2\xa8\x8d\xfc\x8a\x46\xda\x3d\x7b\x8d\xc6\x88\x8e\x3c\x3d\x1a\x89\x54\x36\xbd\x98\x64\x94\xe6\x5e\x78\x3a\x88\x69\x4f\xb1\x11\xe5\x4d\x06\xb1\x14\x3f\x3c\x46\xa2\x5a\x85\x35\xee\x4d\x6c\xbc\xb5\x82\xf4\x62\x3f\xb9\x8e\x58\x5c\x50\x69\x4a\x79\xde\x63\xf5\xbe\x1a\x96\x1e\x03\xf8\xc8\x0e\xd0\xf7\x08\x47\xf2\xa4\xec\x55\xb1\x28\x31\x25\xe1\x44\x48\x52\x6a\xc6\x6e\xa8\x60\x0d\x9e\xb9\xa1\x09\xc0\xe8\x97\xf3\x94\xc1\x7a\x00\xa1\x89\x73\x88\x65\x44\x78\xca\x85\x59\xc3\x39\x8e\xe7\x60\x01\xcc\x09\x32\x12\x88\xca\x6c\x39\x4b\xf3\x1e\xc2\x0c\xa5\x5c\x42\x29\xf8\x9c\xd0\xfb\x94\x11\x34\xa1\x04\xdf\xb2\x5a\x0b\x4d\x23\x9c\xa5\x7c\x15\xed\xb5\x04\x26\x38\xda\x65\x92\xe6\x89\xfa\xff\xe9\x1d\xc9\x39\xd3\x2a\x74\xbd\x51\xa7\xcd\x08\xff\x60\x22\x4a\xb7\x9b\x18\xb5\x08\xd4\xf5\xc8\x0d\x4b\x05\x97\x92\x8e\x91\x46\x28\xb0\x42\xa3\x14\xff\x07\xe6\xdc\x57\x17\x30\x4e\xca\x7a\x09\xf8\xf6\xf5\xa3\x7d\x5c\x26\x04\xe5\xba\xdd\x93\x20\xeb\x74\x23\xe2\x88\x07\x84\xb7\xf4\x74\xc8\xa8\xbd\xd5\x12\x96\x4e\x15\x6a\x1f\x89\x47\x2b\xd6\x25\x4a\xf3\x97\x94\xe2\x55\x28\xca\x7b\xce\x10\xbb\xc2\x5c\xb7\xac\x75\x88\x43\x54\x50\xc0\x6e\x52\x4b\x39\x3a\x46\x8e\x4d\xaf\x68\x07\x7b\xef\x6b\xab\x67\x68\x63\x3b\xb6\x9d\x80\x98\x6d\x61\xc6\xdb\x3d\x7c\x2e\x1c\x15\xa3\x59\xdb\xdc\x8e\xac\x1a\x32\x4e\xa8\x1e\x3a\x24\x7d\x04\x20\x1e\xe6\xaa\xc1\x36\x8b\x16\x53\x46\x4e\x84\x21\x2f\x51\xad\x74\x96\x60\x8d\x4b\xf2\xc0\x2b\x5e\x83\xa2\xf3\x53\xb5\xbf\x3d\x27\xb3\xd3\x87\x32\x0c\xfe\x2b\xbc\x3a\xe8\xff\xe5\x7a\xbf\x1b\x5e\xad\xee\x93\xf9\x82\x5d\xef\x77\x9f\xca\xc5\x5b\x34\x92\x8b\x93\xe0\x39\x03\x31\x82\xb2\x50\x81\x33\x67\xcb\x4f\x54\xd5\x2e\xfa\xa2\xad\x43\x13\x04\xae\x5e\xe9\x59\x7b\x32\x46\xdf\xd4\xbc\xf0\xdf\x1d\x68\xe7\x81\xe8\x15\xe6\x0b\x8d\x11\x0c\xef\x2c\xe7\x1a\xc0\xd5\xe1\xb5\xc1\x6c\x99\xa7\x62\x29\xd1\x6f\x5e\x5c\x5b\xe4\x93\xed\x9f\x37\xa3\xeb\xad\xbb\x0f\x57\x02\xc0\xf5\x0e\x56\x89\xe5\x1d\xdc\x59\x88\x81\x38\x17\x6a\xd3\x56\xb9\xff\xab\xb9\x92\xab\x73\x15\x08\x69\x05\x3b\xf9\xec\xd9\x0d\x57\x26\x7c\x56\xad\xa0\xb9\x83\xc2\x91\x0f\x85\x0d\x40\xc1\x6a\x75\xcf\x7b\x6b\xb8\x6e\x69\xdc\x38\x35\x6b\x7a\x79\xd0\x06\x27\x73\xb5\x71\xb4\x37\x1a\xeb\x5d\xbc\x40\x8e\x3b\xf7\x5f\x3f\x61\xdb\x67\x0a\xf5\xd1\xa1\x98\xd5\x63\x39\xbb\xfd\x7e\xeb\xac\x1d\xff\xff\x33\x6b\x33\xc2\x4f\x4d\xa8\xd8\xf6\x29\x03\x85\xe3\x04\x98\x7d\xfd\x8a\x9c\x02\x17\x6b\xaa\x03\x1e\x95\xc7\x59\xe9\x1a\xf7\x6c\x79\x7b\x88\xd5\xf6\x4d\x8c\x58\xf0\xe9\xc5\xe3\x06\x63\xc5\xdd\xc8\x33\x1b\xd3\xdc\x0a\x37\x64\x55\xa1\x09\xa5\x51\xe8\x27\x70\xc5\x6e\x0b\x62\xcc\x8b\x13\x80\xda\x78\x4b\x69\x17\xb2\x28\x84\x76\xd4\xa4\xa7\xb9\xe7\x38\xb6\x85\x2c\x39\xb9\x57\x28\xab\xa9\xd3\x04\xb2\x89\xac\xc4\x50\xd5\x85\xfd\xfa\xce\xf2\x8b\x06\xe8\x45\x0f\x75\x94\x7f\xad\xe3\xa5\xb7\x02\x6c\xbd\x73\x59\x7f\x47\x85\xf4\xcf\x1e\x37\x5b\x4e\x38\xc5\x31\xff\x3f\x6a\xf0\x33\xc2\xdf\xe9\xe0\xbc\xc7\x88\xb5\x8a\xe8\x33\x52\xad\x42\xb7\x1e\x2b\xd4\x3b\xc4\x8e\xed\x2e\xd3\x8f\x18\x88\x47\xa4\xdf\x59\x68\x6a\x22\xab\xb2\xdf\x2b\xd0\x4d\x84\xb6\xcb\xf3\xee\xa1\x7a\x3b\x8a\xf3\x23\xa9\xb2\x99\xb3\x35\x91\x1a\x02\x7d\x78\xd0\xc6\xa8\xaa\xc9\x3f\x48\x48\xff\xf9\xa3\x31\x62\xfa\xcf\x1e\x92\x55\x7b\xf7\x4b\xa9\x71\x46\x30\x95\x2e\xbe\xae\x5b\xa8\x0f\x48\xba\xb5\xf5\xb7\x61\x21\x54\x6b\xff\x7a\xaf\x1e\x19\xc0\xe6\xc5\x7d\xe8\x89\x29\x8f\xc8\xa2\xe4\xab\xd0\x8e\xb3\xc4\x94\x6f\x38\x8e\xfb\x47\xd8\x6d\xea\xe6\x5f\xb5\xd1\x33\xdb\x8d\xf6\xdd\xaf\xbe\x19\xa4\x37\xd5\xd7\x41\x57\x8f\xfe\xeb\x57\x79\x3c\xb5\xc0\x0f\x21\xfc\x67\x9a\x15\x05\x75\x2d\xba\x01\x7a\xf1\xed\x41\xb7\x87\x0e\xad\x0d\x56\x63\x5f\xb9\xdb\x96\x53\xb5\xaf\x02\xec\x1b\xb6\x83\x75\x64\x09\x3d\x69\xd6\xae\x6b\x24\x53\xcf\x3e\x95\x85\xd1\xff\x3c\xa7\xce\x99\xac\x2e\x8c\xf0\xa4\xa0\x96\xca\x85\xfd\x18\xcd\x4c\x8c\x94\x3c\x70\xd0\x8f\x25\xa6\x78\x51\x5d\xa9\x0d\x00\x4a\x30\xac\x6f\x90\x4d\x48\xb7\xac\x2f\xc3\xcb\xd0\xb8\x25\xea\x0e\x7d\x8f\x3a\x9c\x2e\x09\x04\xec\x80\xcb\x55\xca\x90\x6a\x5c\xbf\x7e\x66\xc1\xd9\x14\xff\xd3\x84\xb8\xe9\x32\xb4\xf1\x3d\xa8\x4e\x81\x69\xd1\xd8\xcc\x49\xdf\x61\xcf\x91\x5d\x55\x5e\xd4\x50\x15\x47\x2e\x10\x22\x46\x5d\x31\xa6\xf3\xb6\xcd\x47\xe1\xe1\x15\xd9\x6e\x49\x33\xb1\x2b\xd9\x70\x06\x2d\x23\x64\x02\x15\x03\x21\xa7\xce\x56\x18\x9e\xf3\x26\x3b\x6a\x08\xd4\x8e\x26\x66\xb3\xf2\x48\x46\x78\x39\xe1\x64\x6a\x2c\x5c\x8a\xb7\xe4\xcb\x5d\xb1\xfd\xdd\x78\xbe\x92\x11\x53\xdb\x31\xb5\x22\xce\x24\xdb\xca\xff\xd4\xdc\x62\x3f\xcf\x29\x1a\xb7\x9d\x09\xd6\xf4\x87\xbc\xc7\x25\x5f\x06\x5d\xe7\xac\x70\x49\xb3\x6d\x27\x80\xa2\x7c\xa8\x90\xf8\x57\x9f\x0a\x42\x2b\x38\x16\xda\x72\xfa\xd7\xe8\x4a\x1d\x8d\xb3\x16\xf0\x7a\x35\x71\xeb\x7a\xcf\xc9\x1c\x3f\xac\x9c\x4b\xf5\xe4\x1e\x63\x19\xaa\x84\xe6\xa4\xd1\x9d\xdf\x6d\xc7\x5d\x0f\x73\xda\x83\x90\xd2\x3a\xed\x44\x19\x7a\x32\x46\x01\xa8\xbd\x1a\xc5\x40\x09\x53\x6a\x93\x47\xb4\x79\x98\xd3\x48\x2b\x1f\x08\xd8\x7e\xe2\xcb\x9e\xa0\xff\x11\x2a\xb8\xa9\xde\x46\x52\xde\x86\xec\x09\x01\xb6\x1b\xcb\xf9\xbd\x24\x0f\xdc\x69\xb4\xf5\xd4\x97\x3c\x90\x78\x09\x17\xff\xd5\xa9\x63\x80\xf6\x05\xd8\xc6\x21\xbb\x45\xbd\xb8\x58\x94\x19\xe1\x64\x67\x02\x8e\x5b\x08\xd8\xce\x4c\x60\x45\x57\xce\x4d\x6f\x58\x4e\xbf\x32\x15\x46\x4e\x43\x5e\x70\x9c\x89\xe2\x0b\x19\x88\x0f\x39\x3c\x36\xcd\x90\x8c\xa0\xdf\x30\x4d\xad\x8d\xd4\xc9\x91\x10\x5e\x58\x17\x02\x16\xe3\x0c\xd3\x46\x60\x4d\x13\xa5\x43\xcf\xe4\xa6\xd3\x8d\xbd\x00\x86\xf9\x32\xcb\xb6\x43\xdf\x04\x46\xbb\x0d\xbd\x7c\xb2\x76\x5d\x3d\x95\x99\x36\xe7\x8b\x2c\x0c\xde\x16\x58\x86\x8b\x48\x46\x31\x53\xb4\x8f\x82\x05\x43\x47\x13\x8a\x06\xc7\xa8\x5a\x88\x64\x2d\x6b\xb9\xda\x47\x81\xae\x26\xde\x04\x97\x02\x73\x19\x7f\x22\x6f\x4d\xc8\x16\xb5\x01\xd5\x8f\xee\xea\xd1\xa6\x15\xea\x3b\x1c\x36\x1b\x11\xb0\x57\x90\x05\x9b\x6d\x71\x86\x88\x16\x91\xd0\x29\x50\xb7\x56\xae\x8d\xdb\x6d\x61\x6a\xc6\xc4\xde\xdd\x2c\xb7\x3a\xee\x74\xea\xfd\x6a\x02\xec\x30\xe4\x9f\xcc\x5d\xd7\xdd\x07\xad\xb4\xb3\x9c\x7b\x67\xd8\xfa\xcd\x63\x06\xee\xc1\xe0\x11\xdd\xdb\x83\x37\x2f\x76\x1b\xbe\x73\x1b\x71\x87\xee\x6d\xc3\x4f\xb0\x66\xb1\xe4\x67\x27\x5a\xe6\xee\xd3\x3c\x29\xee\xe5\x88\x2e\xe5\xcb\x7a\x4d\xb3\x01\x4a\x6b\x77\xfc\x7d\xdb\x93\xda\x95\xca\x6a\x8f\x02\x1b\x2d\x0d\xc1\x3d\x5a\x31\x37\xd9\x75\x97\x68\xac\xf1\x62\x52\x3d\x0a\xac\xfc\x41\xa6\x1e\xe7\x2d\x94\xd7\xaf\x6c\xee\x41\x26\x0d\x33\x82\xe7\x2a\x3d\xd7\x76\x6a\xcb\x7c\x35\x6f\xf1\x84\x64\x8e\x91\x06\x41\x50\xac\x22\x39\x3c\x5f\x40\x14\x29\x53\xa9\xac\x2c\x87\x3a\xbc\x45\x69\x8e\xec\x66\x92\x28\xf2\x95\x58\x94\x75\x44\x95\xa5\x6e\x6d\xa8\x51\xb9\x64\xf3\xb0\x0a\x14\x40\xfb\x0a\xec\xbe\x15\x21\xa0\x56\x3c\x16\xe3\x92\xbc\xb9\x7c\xf7\x56\xe1\x79\x05\x7f\x4c\x00\xcf\xda\xf5\x30\x65\x7a\x74\x6e\x80\xa0\x2c\xfe\x35\xa8\xba\xd2\x98\x7c\x2e\xd2\x3c\x0c\x8e\x26\xf4\x38\xe8\xca\xee\x21\x82\x6e\x2b\x31\x65\x4c\xcd\x65\x71\xc9\xde\xcb\x13\xd5\x56\x72\x72\x5d\x43\xbd\x89\x34\x71\xc4\xee\xb4\xd3\x81\x5e\xbf\x04\xa3\x4d\xc4\xdf\x4a\xfd\xed\xe4\xf7\xd0\xdf\x90\x7c\xfc\x6b\x60\xe8\xa2\xe9\x2b\xca\x7f\x0d\x4c\xc4\x10\xac\x3e\xe2\x47\x8d\x66\x7f\xec\x23\x63\x4f\xd2\x70\x1d\x58\x8e\x33\xd9\x60\xb7\x53\xd3\x1f\xd5\x19\xa3\xa1\x25\x1c\x1a\x56\xa4\x94\x12\x0b\x55\x5f\x67\x05\xe6\xea\xbd\x16\xca\x94\xbd\xc7\xef\x45\x99\xf1\x7b\x0c\x06\x28\xd8\x3f\xcb\xa7\x41\x0f\x05\x7d\xf5\x17\x9e\xd1\x7d\x9a\x65\x68\x42\x24\xb0\x44\x88\x53\x81\xde\xe3\xf7\x68\xb2\xb2\xe1\x77\x23\x74\x39\x27\x1a\x54\x8c\xf3\x0e\x17\x8d\x20\xb6\x9c\x24\x3d\xc4\x0a\xb8\xf1\x8b\xf8\x9c\x2c\x10\x66\x68\x86\x4b\x86\x42\xb0\x04\x22\xdb\x1f\xaa\x53\xc4\xad\x9d\xf3\xd0\xad\x44\x71\xee\x0c\xd6\xf7\x55\x1b\x9d\x60\x25\xce\x08\x37\xd7\x86\xcf\x55\xc6\xba\xe8\x55\x91\x15\x34\xfa\x28\x5f\x56\x6e\x23\x30\xce\x2d\x83\x49\xf0\xd0\x02\x73\x9a\x3e\x04\xae\x8a\xaa\x8c\x54\x15\x1f\x97\x32\x94\x17\x1c\x15\x53\x24\xeb\x43\x44\xc7\x13\xf4\x31\x23\x98\x11\x95\x9c\x08\xa3\xb8\xa0\x94\xc4\x1c\x12\x59\x10\xc6\xd2\x22\x37\xc1\xa2\x8a\x1a\x92\xcf\xd7\x95\x9b\x16\xeb\xe8\x44\x6a\x42\x5e\x2a\xbd\xc9\x59\x3d\xa4\x61\x64\x9e\x24\x17\x57\x31\x0d\x9c\x29\x59\x05\x33\xd0\x35\xd2\x54\x30\x84\xb6\x0d\x47\xb6\xaa\x62\x56\xe4\x54\xcd\xc4\xd7\x31\x14\x95\x6a\x92\xbe\x21\x47\x25\x54\x1d\x57\xb1\x86\x06\xb0\x79\x57\x29\x31\x43\x0a\xbb\x97\x21\xfc\xf6\x9c\xe6\x43\xf5\xd7\xdd\x8b\x72\x26\x23\x2a\x98\x4b\x29\x4b\x80\xe4\xbf\x5a\x27\xe2\xdf\xc3\x50\x1e\xce\x5f\x1d\x5c\xdb\x11\x5b\xab\xa1\xb5\x36\x82\x64\x4a\x68\x57\x87\xd7\xdd\xca\x2a\xad\xa2\x89\xaa\x4d\x48\x26\xb6\x70\x8a\x03\x23\x78\x0c\x65\x0b\xb9\x99\x07\x72\x80\xd9\xdb\x08\xeb\x62\x96\xe0\xca\x88\x60\x98\x31\x06\x0a\x10\x67\x19\x5a\xa4\x8c\x09\x53\x85\x71\x52\xb2\xa8\x62\x01\x72\x6f\x2c\x6c\xa5\x32\xa5\x18\x14\xd6\x26\xc3\x28\x51\x6e\x2d\xfb\xc6\x47\x34\x42\x1c\x1d\xb9\xe5\x24\x4f\x44\xe9\x7e\xbd\x36\x29\x9d\x40\xc0\x97\x59\x56\xdc\x03\xf4\xa9\x50\x1a\x02\xbd\xb2\x48\x73\x8e\xd2\x5c\x46\x74\xc7\xab\xc8\x3e\xc4\x95\x26\xbf\x89\x96\x11\x38\x3e\x7b\x86\x64\xf1\x55\x59\xb0\xeb\xe8\x01\x1d\x89\x7e\x1b\xdd\x4a\xaf\xa0\x3d\x9d\x66\xe0\x52\xa5\x5b\x40\x2c\xd3\xbc\x2c\x20\x73\xa1\x9a\xa8\xfa\x76\xb5\x06\xe2\xcb\xc3\x10\xf1\x1e\x52\x61\xae\xeb\x6e\x33\x44\x07\x21\x93\xe6\xd2\xb4\xad\x26\xb6\x3a\x2f\xc1\x3b\xda\x7f\x8d\x1c\xa2\x1b\xcf\xa2\x80\x26\x36\x05\xb5\xd7\xcf\x35\xc3\x20\xa3\x0e\x64\xf8\xc4\xf9\x0a\x71\x8a\x63\xc2\x84\x9a\xc2\x39\x22\x0f\xa9\xcc\xa8\x07\x6a\x3c\x72\x53\xb5\x54\x5e\x6f\xab\xbb\x2a\xcf\x4b\x3c\x4f\xb3\x84\x92\x3c\xec\x7a\xc2\x9d\xaa\xba\xb5\x1b\x43\xf0\x02\x32\xc7\x38\x2f\xd6\xf5\x14\x34\x4f\xc3\x8e\x65\xb6\x04\x32\xf7\xcc\xb1\x34\x49\x3a\xcd\x1c\x34\xb5\xea\x2a\xf9\x4c\xb3\x7e\x85\x7e\x23\x0f\xe1\xb6\x4a\xd0\x55\x75\x04\x40\xf2\x44\x1d\x00\xb4\x7a\xb6\x05\xe5\x5f\x15\xf9\x9d\x90\x5d\x5e\xa0\x4f\xef\xcf\x7e\x46\x26\x49\x85\xce\x43\x68\x79\x10\x76\x3f\x19\xfd\xfa\x15\x7d\xf3\x9d\xea\xe1\x70\xae\xd3\x6f\x46\x9e\xd3\x09\x8d\x66\xdf\x74\x64\x86\xb9\x5d\xef\x7c\xc4\x09\xc4\x4f\xab\x64\x01\xf7\x29\x9f\xa3\x34\xbf\x4b\x59\x3a\xc9\x08\x0a\x84\x54\x04\x52\x61\x32\x84\x39\x84\x31\xc6\x10\x99\xbc\xa4\x24\x41\x0f\x7d\x31\x09\x68\x52\x2c\xf3\x04\x03\x00\x92\xb3\x25\x25\x4c\x83\xe7\x73\xcc\x25\xe7\x31\x84\x29\x41\x49\xca\xca\x0c\xaf\x48\x22\x7b\xc2\x68\x9a\x3e\x54\x70\x80\x0a\x4e\xee\xa7\x1c\x97\x25\x04\x5c\x16\xd0\xb5\x89\xf2\x36\xf0\xc5\xc0\x75\x33\xa8\x52\xa5\x25\xa8\xd4\xcf\xd5\x81\xd0\x32\xc7\x15\xd5\xac\x18\x15\x49\xa3\x65\x0e\xe9\x04\x41\x1f\x98\x5a\x0d\xbd\xb0\xae\xc3\x75\xb5\x5b\x1f\x1d\x4a\x6d\xa6\x66\xa4\xd1\x8b\x51\x39\xaa\x82\xb7\x83\xea\x92\xf9\xfb\xe2\x1e\xc5\x94\xc0\x1d\x99\x39\x01\xdb\xc6\x15\xe2\x46\x6e\x5e\xdb\xfa\x91\x59\x10\x24\x06\x2a\x0c\x71\x68\x31\xbf\x59\xff\x64\x06\xc8\x61\x75\x74\x64\x09\x36\xf8\x37\x64\x42\xc8\xb0\xdb\x03\x75\xdc\x53\xdb\xcf\x84\xcf\x37\xb4\xf9\x49\xbc\x07\xe7\xd8\x9f\x0f\x7a\xe8\x85\x69\x27\x77\x65\x84\x0e\x3d\x49\x2f\xbe\x57\x91\xa1\x01\x1a\xa2\x20\x4b\x73\xa2\x3d\xd5\xb0\xfb\x2b\x8b\x0c\x2b\x5f\x8e\x78\x87\xa9\x72\x4f\x6b\x7f\x8d\xe1\x77\x15\xd3\x9e\x8a\x9a\x78\xc9\x8b\xa0\xe7\x10\xf5\x75\x9a\x27\x70\xdb\x88\x11\xc5\x99\x1d\x86\x16\xf8\x61\xb0\x48\xf3\xbd\x96\x74\x1c\x42\xe9\x72\x5a\x99\x16\x83\x01\xfa\x69\x4e\x72\x9d\x77\x43\xd8\x85\x32\x27\x5c\x62\xd6\xe2\x05\x7e\xa8\xd6\xe2\x0d\xb2\xc8\x2b\xef\x92\x93\x1b\x22\x5e\x52\x2a\xcb\xdf\xd9\x90\x64\x7a\x1d\xb5\x82\xf9\x21\x8a\xd2\x8f\x62\x45\xae\xfb\x40\xcd\x8b\x68\x85\x8e\x6b\x1d\x3c\x7b\x86\xec\xd7\x4f\x7c\x0e\xbe\x3a\x4a\x56\x03\x8f\x97\xd6\x2c\xa5\x82\x12\xfb\x63\xb7\xb5\xe2\x76\x7b\xc1\x70\x78\x39\x92\xe4\x5b\xe0\x87\xe7\x87\xd1\xc1\xb7\xed\xd5\xd2\x5c\xd3\xc6\x59\xe9\x61\x06\xe0\xdd\x59\x3e\x4d\xf3\x94\xaf\x46\xb5\x99\xe9\xbb\x2f\x1e\x39\x43\xff\x98\x49\x38\x02\x1c\x77\x21\xbd\x1c\xcb\x46\x82\xfb\xe6\x78\xb1\xe3\xcc\x2e\x76\x9f\xcf\xb5\x95\x1f\x02\xb0\x1a\xc3\x34\xd5\x83\xfe\xfc\x93\x89\xf6\x2b\x7f\x73\xeb\x6c\x8a\xdf\xbe\xae\xe7\xcb\xfb\xd3\x0e\x3c\x3c\x88\x0e\x9f\x87\xe6\x8a\xa6\x28\xec\x0b\x78\xdd\x6a\x53\xb2\xa5\xdb\xad\x10\xd6\xda\xa9\x26\x58\xe9\x41\x99\x26\x4d\xbd\x1b\x81\xf9\x03\x27\x04\x5f\xa4\x96\x19\xfa\x54\xb6\x75\x9b\x7b\xb5\x05\xd6\x2f\x4a\x95\xb7\x02\x93\x7a\xaf\xa0\x29\xc9\xb9\xd1\x94\x64\xaa\xa3\xee\x79\x1a\xdf\xbe\x56\x89\xc3\xe0\x4a\x8b\xcc\x22\xf6\x1f\xef\x7e\xb8\xec\x79\xd6\x08\x40\x47\xad\x11\xf6\x95\x6f\x97\x74\x2a\x29\x74\x35\x8a\x79\x71\x47\xe8\x09\xe1\x38\xcd\xfc\x63\x79\x53\x55\xd8\x6d\x40\x12\x4d\xe7\xf6\x49\x28\x75\x7e\x0f\x3d\xf4\xd0\xca\x55\x9b\x2a\xe4\xa9\x73\xc4\x4a\x9c\x6b\x53\x51\x14\x06\xc7\x1d\xb4\x5f\x1d\xe0\x3c\xa0\xe7\x60\xc0\x75\x23\x5e\x7c\xba\x7c\x25\x1d\x3b\x61\x17\xed\xa3\xce\xd1\x40\xb4\x3d\xee\x8c\x2c\xb0\xec\x1e\xf3\x78\xde\x04\x0c\xe3\xb8\x91\x6f\x03\x99\xac\x63\x1c\x4c\x70\x7c\x3b\xa3\xc2\x24\xea\xab\xdd\x61\x07\x76\x37\xa0\x2e\xa0\x44\x74\x23\x2c\xd7\x66\x47\x71\x91\x73\x15\x23\x21\xbb\xdc\x47\x6a\xb4\x91\xcf\x9f\x06\x86\x99\x74\xaa\x0d\x91\xed\x60\x5c\xa9\x91\xc8\x12\xd3\x85\x15\xde\x05\x15\x26\x14\xc8\xa2\x7b\xb5\x8a\x94\x57\xb8\xf2\xa1\xba\x68\x34\xed\x15\xf0\x46\xe8\xfc\x8f\x9e\x89\x7f\x0b\xef\xbc\xf6\x88\x6c\x66\x0c\x92\x8d\x0c\x61\xf5\x36\x4f\x67\xf3\x4c\x98\x26\x90\x43\xd2\xd3\xe5\x0f\x64\x8e\xef\xd2\x82\x46\x4a\x55\xbf\xd1\x0d\x42\xb4\x13\xeb\x49\xbc\x86\xea\xaf\xdb\x39\x9b\x93\xec\x4e\x1e\x23\xec\xd0\xf3\x25\x58\x07\xbb\x31\x7c\x5b\xaf\x76\x24\x81\xc9\x93\xb2\xd5\x09\xce\xd2\xdf\x7e\xcf\x96\xd3\x55\x53\xf5\x03\x3f\x8f\x26\x30\x9b\x02\x13\x8a\xf0\x7b\x4d\xc4\x0d\x56\x41\xa5\x6e\x76\x08\xe8\xf6\x04\x87\x6c\x09\xd6\xf0\xd3\x44\xec\xad\x15\x16\x2a\x85\x1a\x43\x25\x66\x0c\x6e\x0c\x57\x19\xd6\xa6\x05\x35\xf6\xa0\xdc\xf0\x80\xc3\xd4\x4a\xab\xc6\xf0\x1d\xd9\x53\xbb\x22\x2b\x99\xda\xcb\x7f\x7f\xf9\xb3\xc9\x23\x25\x76\x31\x05\x4d\x08\x95\x79\xd8\xfa\xc6\x27\x8a\x52\x2e\xdd\xb6\x56\x9f\x12\xd8\xbd\xb0\x44\x05\xc4\x25\x23\x54\x6c\xb0\xc4\xfe\x48\x5e\x38\x03\x7c\xec\xc4\xb9\x26\x07\x9b\xf2\x37\x3a\x1b\x45\x7f\xee\x36\x70\xbe\x6e\x75\x47\x78\xbd\xa6\xef\x0b\x40\x13\xdc\x43\x0c\x4d\x85\x46\xac\x79\x42\x9b\x7e\x81\x4b\x3c\x71\x33\x37\xd9\xc9\x82\xac\x13\x22\x93\xe3\x6d\x27\x2e\xa8\x85\xde\xd4\x02\x56\xf1\x4e\x7c\x20\x03\x11\xab\xac\x47\x9b\xb1\xb4\x29\x2d\xfd\xe1\xfa\x80\xe4\x87\x22\x59\x69\x52\x5b\xe0\xdc\x54\xd6\x37\x90\x1c\x00\xf1\x49\x91\xa8\x24\x86\xd0\xce\x89\x52\x64\xf7\x29\x8f\xe7\x61\xed\xfc\x5f\xdd\x53\xc6\x8c\xa0\xe0\x8e\xc4\xbc\xa0\xc1\x70\xcf\x36\x0f\xcd\x25\x58\x25\xd4\x62\x21\x69\x1e\xcb\xbb\x33\xab\xbb\x57\xce\x92\xe0\x88\xd3\xe3\x23\x9e\xa0\xb8\xc8\xc4\x1a\x36\xee\xbc\xe8\x1c\x1f\xa5\xc7\xb9\x9c\xf0\xa3\x41\x7a\x7c\x34\xe0\x89\xf8\xa1\xc7\x41\xed\xe6\xa5\xbd\x95\x76\xf0\x51\x8b\x41\xba\x48\xb9\x54\x9b\x36\xba\xdd\x9a\x57\xd3\xba\x9a\xe4\x89\x29\x70\x2f\xd8\xc3\x64\x2a\x03\x57\xa7\xf8\x48\xaf\xed\x65\xd7\x9c\x5a\xf9\x5c\xdb\xc6\xb3\x3d\xda\x44\x8b\xe3\xda\xf9\x9d\x04\xa9\x4e\xd9\x04\x2d\x54\x15\xe5\xb9\xbe\x3a\xbc\xae\x5e\xd9\x64\x92\x84\x81\x5b\xa3\x23\x33\x91\xea\x78\xc2\x3b\x91\xff\x8f\x4e\xd8\xdd\xef\x9f\xb0\xbb\xfa\x84\x99\x1b\x7d\x97\xe4\x41\xe0\x1d\x98\xc3\x0f\x83\xde\x67\x89\xde\x67\x74\x84\xee\xf4\xd9\x82\xc6\xed\xb3\x9b\xad\xa1\x82\xb4\x3f\x36\x95\xaf\x3e\x5f\xab\x29\x45\xff\x43\x4c\xb3\x5d\x7e\x20\xa7\x7a\x42\x07\xc7\x81\xeb\x60\xfe\x83\xbc\x64\x61\xb2\x33\x2b\xa9\xd3\x1f\xc9\x4a\xfe\xde\x65\x15\xa7\x27\x7b\x26\xda\x38\xb7\xde\x11\xd8\xd4\x9b\x3b\x82\x2a\x4e\x47\xd6\xa8\xdd\x3e\xbb\x5b\x3a\x55\x0e\xd2\xa1\x77\x25\xfa\x94\xb3\x65\x59\x16\x94\x93\x44\x5d\xcd\x84\x93\xbb\x06\x90\xf5\xef\xb6\xb3\xfc\x9f\xd7\xf2\x25\x71\xa9\x7f\x17\xc7\x71\x92\x5b\x9d\x9f\xfb\x8b\x77\xc6\xa9\xda\xdf\xd9\x78\xad\x2a\xc4\xf0\x84\xdd\xac\xec\x94\x46\x2b\xb3\xce\xcb\x57\xc7\x63\x74\x48\x5e\xfc\x5b\xed\xb6\x4b\xb8\x42\x03\x59\x1e\xf1\xc2\xda\x38\x05\xbf\x04\x23\x37\xbb\xb2\x0d\xe5\xb0\x05\xca\x61\x1d\xca\x7f\x6e\x80\x72\xf8\x67\x3f\x94\xc3\x3f\xd7\xa1\x9c\x6e\x82\xf2\x6d\x0b\x94\x6f\xeb\x50\x3e\x6e\x82\xf2\xa2\x05\xca\x8b\x3a\x94\xcb\x0d\x50\xfe\xe2\x07\xf2\x97\x3a\x8c\xbf\x6e\x80\xf1\x9d\x1f\xc6\x77\x75\x18\xef\x36\xc0\xa8\xdf\x86\x56\x30\xbe\xa9\xc3\xb8\x6d\x87\x51\x83\xb0\xf2\xd5\x73\xd6\xa8\x4d\x15\x8f\x04\x52\xfd\x36\xde\xeb\x37\x99\x6f\xe5\x47\x4c\xc1\x69\xe1\xbe\x7e\x93\xfd\x7e\xdb\x04\xa7\x8d\xff\xfa\x4d\x06\xc4\x1b\xe1\xb4\x70\x60\xbf\xc9\x82\xd3\x8d\x70\x5a\x78\xb0\xdf\x64\xc2\x72\x13\x9c\xbf\x54\xcb\x5b\x0d\x50\x83\x11\xf3\x4d\x70\x5a\x38\xb1\xdf\x60\xc5\xff\xf5\x3f\xdb\xc0\x1c\x92\x7e\x0b\x2f\xf6\x1b\xcc\xb8\x68\xc7\xc5\xc7\x63\x5b\x32\x65\x58\x76\x8c\x93\xc7\x40\x5a\x33\x9b\xc2\x4f\xde\xbd\xfc\xf9\xe6\xe2\xf4\xfc\xec\xf4\xe2\xe6\xfd\xa7\x77\xea\xcb\x91\xd5\xad\x17\xc2\x18\x86\x8b\xe0\x41\x7b\x8e\xb6\xd7\x84\xc7\x73\x2b\x45\x9b\x6b\xda\xed\x43\x02\x36\xe9\xf3\x9a\x2c\xb9\x3e\x45\x4b\xf3\x19\x2a\xf2\x6c\x85\xa6\x29\x65\xdc\xb4\xad\xa1\xb3\x8f\x82\x28\x30\xd1\x84\x2e\xe0\xe3\x5a\xe5\xc6\x4e\x4e\x47\xc9\xab\x41\xb8\x61\x2c\x0a\x16\x2b\xb3\x34\x26\xe1\x41\xaf\x0e\xac\x16\x03\x24\xab\x43\xc2\x12\x99\x46\xa9\x4a\x73\xa4\xae\xe6\x54\xc9\x8e\x86\xe8\x0a\xbc\x41\x72\x27\xad\x9f\xea\x89\x94\x86\x32\x07\x24\x9c\xa7\x86\xde\xac\x92\x1d\x95\x2e\xb2\x23\x4f\x54\x05\x98\xda\xb2\xf9\x11\x37\x2e\xe9\x7b\x6a\xd4\xb2\x44\xf9\x57\x7a\x40\xb6\xca\xb6\x02\x5e\x56\xc0\xed\xd3\xf9\xdb\xea\x8c\xdd\xae\xe5\xb5\xdd\x9d\x0a\xf2\xc8\x70\x5d\x05\x73\x3a\x6f\xf5\xb9\x03\x74\x85\x93\x44\xba\x91\x90\xf9\x4e\xd8\xd3\x30\xf8\x13\x4e\x92\x1b\xf5\x5d\x17\x95\xf6\xd3\xa9\x2d\xbf\x9f\x23\x8a\x7a\xe8\xcb\xba\xdb\xb4\x2b\x6a\xc3\xd7\x03\x6a\x92\x40\x0c\x4e\x85\x7f\x8a\xb9\x80\x3c\xeb\x8c\x60\x2a\x3f\xdd\x16\x04\x35\x99\xd4\x41\x50\x8a\x78\x30\xb7\x1f\xf5\x8d\x17\x3f\x9c\x88\x2d\x27\xd2\x66\x0c\x0f\xbb\xc0\x77\x3c\xec\x3c\xeb\x98\xab\x6c\x15\x8c\x37\x24\x2b\x8d\x5b\xb0\x3e\x98\xbf\xd5\xaa\x85\x76\x28\x47\x1d\x86\x1c\x70\xd5\x84\x85\x16\xa6\x5b\xa9\xa5\xa9\x6c\x53\x4b\x7f\x33\xd1\xe5\x9b\x26\xae\xd2\xc5\x01\x24\x7b\x6a\xbe\x57\x68\x7d\xb1\x4a\x39\xfc\xd5\xd7\x1c\xa5\x99\x2d\x66\x56\x3a\x48\x3e\x9d\xbf\xad\xa6\xb6\x6b\xbd\x96\xe6\x66\x6d\xee\xbb\x7b\x90\x38\x71\xcf\x49\x9e\x27\x85\x4f\xb2\xa0\x8e\x9f\x00\xa6\x92\xb3\xd3\x55\xee\xc5\x66\x84\xaf\x8e\x0a\x31\xce\xc7\x2a\x5f\xb5\x20\xd7\x60\x80\xde\x7f\xb8\x3c\x1d\xd6\x72\x2d\x4d\x08\xba\x25\x25\x87\x8c\x54\xab\x3c\x96\x11\x02\x83\x25\x4f\x33\xa1\x26\xf5\xdf\xb8\xc8\xef\xa2\x59\x31\x04\xb8\x6f\xd3\xfc\xf6\x75\x41\x4f\x4d\xa4\xdd\x86\xa9\x30\x64\xf1\x0b\x2f\xcc\xaa\x5c\x5d\x20\xea\xb2\x4e\x05\x27\xd0\x6c\x26\xe5\x0c\x12\x0f\xd9\xc1\x79\x35\x0d\x20\xe9\x50\xa5\x60\xd2\x11\x32\x7f\x98\x57\x2d\x10\x1f\x26\x9f\x49\xcc\x75\x2e\x3a\x9b\x71\x67\x24\x27\x14\x73\xc9\xbb\xb2\x9a\xa3\x7c\x34\xfe\x8e\x4e\x7f\x2a\x23\xb0\x42\x0b\xb6\x0e\xc2\x96\xdf\x47\x94\xb1\xaf\xcf\xd4\x27\x96\x94\x7a\x05\x1e\xb9\xe0\x98\x93\xf0\xcb\xba\x87\x82\xa0\x87\x64\x3c\xcf\xf7\x62\x4f\x67\x91\x76\xab\xc0\x58\xdc\x69\xcf\x93\x64\x3f\xa0\x74\x63\x6a\xbc\xd3\xa5\x52\xbb\x55\x00\xba\xe8\x8b\x1a\xe2\x0c\xfc\xd9\x50\xcf\x73\x4f\xc3\x4b\xf5\x1d\x16\x88\x7a\x93\xba\xca\xfc\x9b\xa3\xdf\x0c\x34\x5b\x99\x18\x5e\x04\x0f\x30\x49\xdc\x26\xf2\xd0\x0e\x86\x75\x96\xdf\xe1\x2c\x4d\x3c\xfa\x48\xa6\x60\xb3\xf5\x99\x6c\x26\xcc\x0b\x35\xed\xaf\x69\xb1\xf8\x20\x3b\x50\x00\x9a\xdd\xf5\xd0\xc1\x8e\x94\x89\xaa\xde\xe5\xe9\x22\x1a\xa3\xc1\x7f\xcd\x7e\x4d\xf6\x7f\x8d\xa2\xfd\x71\xb4\xff\x74\xf0\x38\x62\x79\x46\x68\xd3\x0b\xb8\xf3\x72\x59\x66\xfa\x38\x5e\x0d\xd3\x2a\x6f\xcc\x7d\xf5\xae\xb6\x04\x3d\x7a\x70\x11\x27\x8c\xdb\xf0\x46\xfe\xcb\x3e\x5b\x07\xb9\x69\x3e\x5a\xd8\xa3\x27\x59\xf6\xac\xd2\x39\x62\xc1\xb5\x2a\x54\xc6\x44\x63\x8f\x55\x5b\x6b\x4b\xf8\x9c\xf0\x87\xa9\xd0\xbf\x00\xcf\x49\xb1\x08\xd0\xe4\x17\x87\x43\xab\x4b\xbd\xc8\xe6\xcb\xc5\x84\xd0\x0f\x53\xd9\xe9\xeb\x82\x0a\x28\x5a\x60\x6d\x74\x76\x9e\x86\xea\x85\x0c\x4e\x65\x3f\xa5\x7c\x1e\x36\x90\x54\xc4\x36\xf7\xc6\x14\x05\x36\xe1\xb3\x9d\x12\xdb\x06\x51\x19\xb7\xed\xfd\x74\xad\x4b\xe3\x35\x50\xcd\x42\x77\x21\xd9\x89\x26\xc6\xe8\x69\x90\x44\xd1\xc2\xb0\x61\x23\x83\x5f\x65\x83\x5a\xd2\xfd\x61\xfa\x21\x57\xeb\x72\x13\x3f\x33\xcf\x12\xc8\xcb\x38\x5e\x2e\x96\x19\xe6\x70\x59\x6c\x07\x65\xd2\xc2\xb1\x68\x5f\xa5\x1c\x68\x80\x35\xb1\x77\xd5\x97\xa8\xeb\x69\xe8\xac\xda\x8f\x16\xb5\xf6\xc1\x6f\x57\xc3\x4e\x22\x44\xe4\x32\x77\x23\x4c\xc8\x9e\xc4\xaa\xf5\x7b\xbc\x20\x2f\xf3\x44\xdf\xf5\xe0\x72\x46\xa5\xe5\x3a\xee\x58\x8b\x79\x55\xdd\x7c\xe8\xdf\x6e\x0b\x39\xc9\x6b\x95\x35\xd0\x84\xc4\x45\x42\x3e\x9d\x9f\xbd\x2a\x16\x65\x91\x93\x5c\xd3\xd2\x01\x70\x78\x5d\x65\x37\xfd\x75\x1f\xd2\x9b\xa2\xa0\xab\xd3\x95\x0b\x49\xb2\x51\x18\xa3\x80\xe3\x89\x75\xa5\xc6\xed\xd2\x24\xaa\xb0\x8a\xe5\xd7\x0a\x38\x9e\xa0\x94\x41\xcc\xde\x8c\x50\xe5\x87\xb6\x2d\xd5\xab\xaa\x9b\x6b\x33\xd4\x1f\x75\x7e\xc4\xb5\x67\xfa\x9b\x59\x08\xb7\x4d\x7a\x5d\x8f\xd9\x53\x6d\x19\x6d\xaa\x97\x60\x26\xac\x94\x54\xb1\x29\xec\x96\x1f\xd9\x9f\xc7\xd4\x6a\x58\x2f\x35\xab\xcb\x70\x59\xa9\x31\xf4\x6b\xe0\xd4\x51\xbe\xae\xc9\x27\xd9\x52\x3e\x46\xb7\x64\xc5\x9c\x9e\xba\x4d\x26\xbd\xad\x3e\x97\x6d\x41\xba\x52\x28\xec\xa3\x5b\xb2\xba\xd6\x76\xab\x82\x72\x25\xca\x1a\x01\xef\x56\x6b\x63\xd3\x5f\xce\x09\x23\x88\xdf\x17\x2a\x0b\x01\x43\x61\xca\x4e\x48\x49\x49\x8c\x39\x91\xfb\x20\x61\x7e\xe3\x3c\x41\x94\x24\x29\x25\x31\xbf\x2c\xde\xa5\x33\x41\xb9\xe4\xd3\xf9\xdb\xae\x80\x82\x29\x41\x38\x49\x48\xa2\x5c\x1b\x05\x85\x2f\xc3\xde\x63\x9a\xc0\x8d\x6f\xcc\xd3\x49\x9a\xa5\x7c\x25\xb6\x0c\x45\xa6\x73\xc2\x4b\x6f\x77\xb4\x67\x32\xd4\xfa\xba\xde\xb0\x51\x9d\x63\x36\xdf\xb0\x80\x56\x9f\xbf\xd0\x3a\x56\x0a\x5d\xf2\x9a\xe2\x99\xca\x6a\xe2\x11\x43\x5f\x2f\xf2\x34\x97\xae\x8c\x64\x59\x3e\x8c\x1a\x50\xa5\xfa\xc3\xc3\xae\x94\xad\x84\x16\x25\x1c\xec\x0b\x38\xe8\x4f\xe0\xfc\x8a\x21\x4c\x28\x24\x0d\x17\x9e\x85\x72\x65\x0c\x52\x21\x65\xeb\xbd\x8a\x4c\xde\x89\xb0\xb6\x45\x7f\x6c\x98\x9e\x3d\xd1\x1f\x19\xad\x5f\x02\xea\x4e\x11\x67\x81\x2d\x5c\xa9\xab\xd4\xb3\x11\x3b\x8f\xf4\x8b\x3a\xb6\x54\x15\xbb\x08\xd4\x66\x91\x2a\x6a\xd2\xa4\xe5\x69\xed\x8a\x36\x64\xf4\xf0\xef\xc0\x6a\x44\xf6\x64\x29\xa9\xed\xb8\x60\xa2\x07\xcf\x9f\xef\xa1\xe7\x48\x26\x61\x52\x59\x2b\xd0\x5c\x6e\x6a\xf4\x28\x98\xa8\xf1\xfc\xf9\x40\xb9\xe5\xec\x74\x17\xca\x31\x67\x52\x9b\x7b\x72\xb3\x2b\xd6\xda\xea\x89\x93\xdf\x61\xeb\x03\xf4\x7e\xe5\x96\xdb\xd3\x59\xa6\x1b\x09\x8e\x87\xde\xed\x9f\xce\xc1\x2e\xbf\x50\x7d\x9a\xc9\x9b\x21\x49\x7a\x17\xd5\x20\x8f\xac\xca\xea\x23\x68\x4f\x43\xdd\xaa\x5b\xdd\x6d\xe9\xa4\xba\xae\x7e\x19\x15\xd3\x69\xd8\x01\x5f\x59\xc7\x5e\x1f\xdb\xb2\xbd\x5b\x87\xee\x42\x85\xcb\x50\xf0\x0f\x79\xd5\xd9\xc8\x5e\x06\xab\x5e\x72\xdd\x49\xcf\x97\xf1\x1e\xee\x78\xc6\x52\x9a\x64\x94\x49\xc7\xf3\xd1\xb3\x8e\x93\x17\xd3\xc1\xb0\x0d\x0f\xc7\x65\xde\xda\x03\xc0\xdf\x0e\x7d\x3a\xf5\x80\xb7\xf9\x1b\xe6\xd6\x49\x9a\xb2\x79\x5e\xb5\x18\x48\xdf\x89\x9b\xbd\xd8\x4c\x27\xbb\x48\x17\xa5\xfe\x78\x86\xbb\xff\xb7\xbe\xb6\x00\x32\xfd\x61\x1a\xaa\x3c\xf3\x62\xd7\xdf\x3f\xac\xc2\x97\x5d\x28\x75\xfd\xe9\xcc\x59\x46\xb8\xba\x3b\xfb\x37\x57\x38\x1e\xe1\x7f\xb6\x07\x30\xc7\x4c\xd4\x54\xe0\xb4\x3d\xe5\x00\xdf\x86\xbd\x0b\xa2\xc2\xde\x03\x49\xed\x61\xb6\xf6\xa1\xb3\x82\x99\x61\x7b\x5a\x80\x33\x50\x56\x1f\xd5\x67\xed\x55\xb1\x84\x15\xc2\xd3\xcc\xce\xf5\xe1\x25\xa4\x0f\x6d\xc0\xda\x82\xdc\x47\xdf\x1e\xf4\xac\xae\xb4\x6c\x7a\x3f\x14\x65\x28\xdf\xf8\x50\x54\xb3\x2b\x3d\x39\x1b\xbe\xd9\xe0\x45\x71\xf4\x58\xd5\x20\x7d\x90\xe6\x1b\x13\x12\xf6\x05\x71\x3e\x05\xf1\xb8\xaf\x45\xd8\x81\xb8\x20\x6a\x5a\xea\x2d\x29\x33\x02\xaa\x90\xf9\xe7\xa1\xa2\x9c\xd5\x2d\x5a\xd6\x09\x99\xf3\x2b\xb3\xea\x2b\x81\x4d\x55\x54\x53\xd2\x55\xcd\x94\xf5\x0d\x80\xd1\x06\x8e\xf0\xae\x40\x3d\x79\xb3\x67\x54\x27\xe0\x74\xfa\x8f\xa0\xa0\x45\x9e\xdf\x4b\x1d\x35\x7a\x2f\x65\x2c\xca\xd5\xa8\xe3\x00\xfb\x83\x04\x02\x4b\xd2\xa2\x50\xdb\xc0\x2d\x82\x2d\x74\x59\x95\xa0\xae\xe1\xa1\xad\xdb\x52\xce\x62\x60\x9d\x95\xc9\x45\x40\xe6\x0b\x35\x9f\x50\x70\x57\xa6\x4d\xf5\x22\xf3\xb5\x16\x83\x94\x77\xa1\xd2\xdb\x98\x13\x32\x25\x54\x6c\x60\xef\xb4\xa5\x55\x4c\xd1\x34\x87\x9d\xc8\x3d\x4e\xf9\x47\x42\xd3\x22\x11\xe8\xc9\x85\x82\x54\x9f\x73\x10\x1b\xdf\x18\x67\x19\x11\x1b\x9c\x92\x08\x6b\x3a\x5b\x55\xb6\x76\x42\x26\xc5\x32\x8f\x49\x38\xcd\x7b\x16\x28\x2b\x79\x86\x4c\x99\x62\xed\xb3\x1a\x76\x81\x93\xf0\x45\xd5\xb7\xbc\x08\x98\x42\x22\x0d\x4c\x67\x4b\x61\x52\x5b\x1f\x98\x63\x71\x51\x12\xe7\xb3\x73\x15\xd6\xf9\x5d\x71\x4b\x5e\x7b\xbe\x79\x3c\xcd\x23\x5c\x96\xd9\x2a\x84\xd6\x3d\x00\xef\x04\x97\x29\x0c\x20\x10\xcf\x64\x7c\xd1\xf0\x9c\x41\x8e\x64\x76\x45\x7b\xef\x61\xa7\x14\x90\xa7\x03\x86\x12\x24\xe7\x29\x5f\xbd\x93\x5f\x74\x81\x9e\x82\x67\xc1\x10\x05\xcf\xf0\xa2\x1c\xe9\x0f\x47\x1c\x41\x49\xc6\x4d\xc1\x31\x14\xcc\x4c\x41\x27\xe8\x0c\x51\xe7\xd9\xdf\x97\x05\x1f\xa9\x0f\xdf\x07\x9d\x40\x14\xfd\xe9\x9b\xbf\x98\x92\x81\x2c\x79\x78\xf1\x7a\xd4\x31\x29\x20\x15\x01\x54\xb8\x81\x42\xaf\x72\xa0\x5c\x3d\x3b\x3a\x0e\x3a\xbf\x0e\xae\x07\xb3\x9e\xf5\x55\x14\x56\x33\x8a\xcd\x30\xae\x98\xd9\x43\xdb\x14\x48\xf3\x94\x2b\xaa\xcb\x44\x7e\x17\x84\x2f\x4b\x15\x42\x1f\xe3\x78\x4e\xd4\xd7\x7c\x2a\xcf\x88\x93\xf0\xcf\xfb\xb5\x2f\xc6\x31\x4f\xe3\xc1\x67\x26\x37\x04\x37\x9c\x2c\xca\x0c\x73\x1d\x0b\x3d\xc1\xf4\xfb\xbb\xb1\xd8\x25\xfc\xf0\xe9\xec\xed\xc9\xcd\x8f\xa7\xe7\x17\x67\x1f\xde\xab\xdb\x94\x8d\xb4\x7e\x42\x9e\x04\x86\x7b\x96\xa4\x5d\x2a\x88\x2a\x44\x53\x8b\xd3\xbb\x25\xe3\x02\x67\xbd\xbd\x13\x2d\x47\xb6\x35\xeb\xdd\x92\xbb\xf9\x1a\xbc\xbb\xd1\x9a\xdd\x6a\x87\x62\x52\xa4\x0e\xf0\x3d\x87\x67\xa1\x95\x15\x00\x3c\x38\x92\xdc\x35\xe9\xaf\xa6\xe5\x69\x28\x2a\x74\x47\x7b\xff\x3b\x00\x00\xff\xff\xd3\x86\x2c\x7a\x63\x94\x00\x00") func pkgUiStaticJsGraphJsBytes() ([]byte, error) { return bindataRead( @@ -520,12 +520,12 @@ func pkgUiStaticJsGraphJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 37691, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 37987, mode: os.FileMode(420), modTime: time.Unix(1571324761, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _pkgUiStaticJsGraph_templateHandlebar = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5f\x8f\xe3\xb8\x0d\x7f\xef\xa7\x60\xd5\x97\x5d\x14\x9e\xdc\x5c\x71\xf7\x50\x38\x2e\xae\x7b\x83\x03\x0a\x0c\xb6\xd8\xdb\xbb\xd7\x80\xb1\xe8\x58\x1d\x59\xf2\x49\x72\xfe\x34\x98\xef\x5e\xc8\xb2\x3d\xce\x1f\x3b\xce\xcc\x36\xc0\xe4\x21\x9b\x91\x48\x8a\xa4\xc8\x1f\x69\x7a\xa1\xf9\xc4\x5c\xac\x41\xf0\x39\x5b\x19\x2c\xf3\xc5\xc6\x60\x59\x92\xd9\xef\x05\x7f\x7e\x66\x90\x4a\xb4\xf6\x68\x8f\x25\x7f\x82\xee\x13\x67\xda\x14\x2d\xd9\x1f\x15\x99\xdd\xc2\xaf\x1c\xd0\x34\x87\x34\x44\x7e\x3b\x32\x7a\x73\x44\x72\x48\x94\x6a\x19\xc9\x55\x74\xff\xdd\x09\x15\x40\xec\x68\xeb\xd0\x10\x82\xd1\x1b\x3b\x67\xf7\x0c\x4a\x89\x29\xe5\x5a\x72\x32\x73\xf6\xb0\x2d\x0d\x59\x2b\xb4\x82\x0f\xf5\x2f\xf8\x35\x17\x99\xfb\xeb\x83\x72\x64\x20\xd3\x06\x14\x6d\xa4\x50\x64\x3f\x32\x50\x58\xd0\x9c\xd1\xb6\x34\xac\xf6\x82\xff\x75\x64\x7c\xad\x70\xaa\x95\x33\x5a\x02\x75\xc2\x17\x42\x95\x95\x63\xc0\xd1\x61\x54\x1a\xbd\x16\x9c\xe6\xcc\xed\x4a\xc2\x9c\x90\x33\xc0\xca\xe9\x54\x17\xa5\x24\x47\x73\xa6\xb3\x8c\x81\x2d\x49\xca\x34\xa7\xf4\x69\xce\x32\x94\x96\x58\xb2\xdf\x7b\x91\xcf\xcf\xf1\xac\x35\xeb\xc4\x2f\x33\x2e\xd6\x13\x9c\xf5\xfd\x39\x5f\xf5\xc8\x68\x8d\x72\x61\x1d\x3a\x0b\x99\xd4\xe8\x22\x23\x56\xb9\x63\xc9\x59\xf9\x00\xb1\x28\x56\x60\x4d\x3a\x67\xfb\x3d\x94\xe8\xf2\x7f\x1b\xca\xc4\x16\x9e\x9f\x67\x5e\x88\x48\x67\xa2\x58\xcd\xf0\x3f\xb8\x8d\xa4\x46\x4e\xe6\x6e\x25\xb2\x7f\xac\xe7\xfb\x3d\x2c\x2b\x21\xf9\xef\x64\xea\x3b\xe8\x79\xd2\x96\x42\x29\x32\x0c\x50\xba\x39\xf3\xac\x8b\x76\x69\x82\xd1\xe7\x96\x8e\xe3\x4a\x28\x7f\xb1\xa7\xd2\xea\xbb\x6a\x29\x97\x4e\xc1\xd2\xa9\xa8\x34\xa2\x40\xb3\x03\xda\x52\x5a\x39\x5a\x2c\x9d\x62\xe0\x2f\x70\xce\x6c\xb5\x2c\x84\x63\xb0\x46\x59\x91\x0f\xa9\x9a\xa2\x0d\x97\x66\xf7\xe4\x14\x4b\x92\xd2\xee\x98\xb4\xb2\x4e\x17\x51\xb3\x38\x14\x45\x61\xbb\x95\x2c\x94\x25\xe3\x16\x05\x39\x23\xd2\x73\xf7\xa9\x4b\xe7\x9d\xda\xe8\xc5\x92\x08\x02\x0b\x04\x16\x40\x07\x69\x65\xac\x36\x10\xc5\xb3\x40\x7c\xea\xda\x70\xe6\xc9\xfa\xb2\x72\x4e\xab\xc6\x03\xe1\x0f\x76\xec\x33\x4e\x19\x56\xd2\x41\x40\x04\x5d\x3a\xef\x35\xe0\xc4\xab\xb2\xf6\xdf\xa9\xe7\x3b\x0c\x91\xbb\x32\x17\xa9\x56\x3e\xe2\x44\x12\x78\xa4\x48\xd1\xeb\x78\xa2\x62\x38\x7e\xe0\x1e\x83\x86\xb9\xe0\x9c\x54\xeb\xb9\x5a\x5c\x77\x61\xf7\xa7\x8a\xbc\xde\xba\x12\x8d\x13\x28\x17\x86\x6c\xa9\x95\xa5\x2b\x0d\x6d\xd8\xa1\x65\x7f\xbb\xad\xc7\x0a\x0d\x9a\x3d\x25\x65\xa6\x42\xf1\x45\x78\x31\x46\x1b\x40\xe9\x83\xb1\xfe\x8e\x38\xaa\x95\x4f\xed\x21\x80\xe9\x31\x6f\xd0\x28\xa1\x56\x07\xec\xcd\xda\x00\xff\x30\x40\x1c\xae\xfd\x39\x8a\x8e\x38\xbf\x7e\xfe\xf9\xf3\xdf\xe1\x93\x56\x6b\x7f\x96\xcb\x85\x05\xa7\xe1\x9f\x5a\x3b\xeb\x0c\x96\xa0\x70\xbd\x44\x73\x07\xf0\xd5\x6f\x19\xfa\xa3\x12\x86\x2c\xfc\x0b\xd7\x68\x53\x23\x4a\x77\x62\x89\xff\x18\xca\x0c\xd9\xfc\xee\x68\x33\x8a\x6e\xe4\x7e\x29\xac\x8b\x56\x46\xfb\x24\x30\x5a\xfa\x52\x84\xcb\x12\x15\xc9\x33\x8c\x00\x71\x25\x5b\x4e\x85\x6b\x6f\x73\xe4\x70\x69\x7b\xbc\x5e\xe0\x59\x56\x80\x58\x8a\x1e\x73\x24\x1c\x15\x2d\xa3\xc7\x35\x52\xae\x4e\x6a\x96\xc4\xd8\xa7\x93\x42\x3d\x31\xc8\x0d\x65\x73\xf6\x97\x3a\xc5\xda\x4a\x8b\x46\x60\x0b\x8e\x6d\xbb\xd1\xee\x75\x0a\x35\xa5\xd6\xe9\xd5\xaa\x5d\x49\x7e\xf1\x94\xf1\x0c\x93\x78\x26\xc5\xdb\x94\x6d\x89\x30\x75\x62\x4d\xa3\xba\xa7\x5a\x59\x2d\x69\x40\xfb\xa3\xdd\x51\xfd\x3f\x05\xda\x31\x0b\xe2\x59\x25\xcf\xae\xf7\x2e\xdf\xe1\xb2\x56\x80\xd4\xe0\x95\x79\xea\xa3\xb8\xe8\x73\xfb\x95\x06\xf6\xbc\x20\x14\x8a\x0c\x18\xf2\x95\x9d\xbd\xb4\x87\x8d\x4d\xe7\x8f\x38\x8a\x5b\x49\x68\x32\xb1\x1d\x24\x0e\xb9\x09\x0f\x5b\x67\x30\x75\xc4\x7d\x12\x66\xda\xa4\x5e\x0d\x5d\x95\xc4\xa1\x06\x40\x7b\x77\x92\x43\x43\x47\x96\x46\x17\xe4\x72\xaa\x6c\xe8\xcd\x16\xb5\x20\x30\x1e\x86\xc2\x0a\x94\x95\x94\x91\xa4\x6c\xc8\x4d\x8d\xd0\x80\xc8\x23\x14\x70\x5c\x39\xa4\xef\xa5\x7a\xe2\x47\x79\x0f\x6a\xd0\x28\x65\x5b\xdb\xd2\x45\x6d\xc5\x05\xb1\xc2\xf9\xfb\xfd\x35\x37\x42\x3d\x81\xcb\x09\x9c\x28\x28\xd8\x7f\x37\x6a\xf0\xd9\xf2\x05\xdd\xaf\xa8\x10\xaa\xb2\xa1\x9c\x8d\xb9\xad\xad\x64\x67\x40\xf7\xe5\x13\x45\x49\x28\x6d\x53\xdc\xdb\x39\x34\x04\xc3\xb8\xfd\x3e\x4c\x7b\x97\xdd\x04\xeb\x14\x97\x7d\xed\xfc\x04\x3a\x0b\x69\x30\xe5\x06\x7d\xc7\x3e\xe5\xfe\x7a\x4a\x8d\x93\x5b\xf1\x5f\x9a\xb3\xbf\x8d\x13\x35\xb5\x7e\xbf\xef\x89\xf5\x49\x79\xd1\xeb\x13\xa2\xfa\x6d\x71\x7d\x4d\x64\x43\xd7\xf1\x4e\x8a\xed\xee\xaa\x7e\x31\x7a\xf3\x4d\x63\xbb\x94\xd7\x84\xf6\x30\x10\x9d\x69\x3e\x0e\xf6\x6f\x03\x76\x7d\x80\x7b\x23\xc2\xdd\x3e\x16\x3c\xce\x91\xe2\x13\x23\xe1\x0b\x6d\x84\xe2\x75\x2c\x90\xff\x57\x14\x6f\x8c\x84\x25\xa6\x4f\x1b\x34\xfc\x4a\xa0\x1b\x22\x9c\x06\x74\x67\xa0\x8e\xa3\x6b\xeb\xd5\x04\xcc\x08\xb8\x47\x8a\x4f\xc1\xbb\xce\x79\x0f\x8d\xc7\x3a\xbc\x83\x0f\xbf\x7d\xfd\xf4\xf1\x12\xf7\xc1\xa0\xe5\x37\xe5\x84\xbc\xc4\x51\xf7\x3c\xbe\xd3\x45\x37\x67\xbb\xdd\x6e\x17\x3d\x3e\x46\xfc\xf2\x1d\x4f\x04\xd8\x36\x74\x48\xf1\x29\x00\xdb\x42\xec\xfd\x8f\x97\xe8\x3a\x94\x25\xc5\x3b\x74\x7d\x9f\xf0\x3a\x3d\xa5\x7e\xe2\x6b\x54\x29\x7d\xc3\x9c\xca\xb4\xb9\x2e\xa5\x5e\x0f\xb0\xd7\x81\xe3\x98\x41\xfd\x61\x51\x33\xe5\xeb\x30\xc7\x6a\x59\xd5\x23\x18\xa1\xc0\x52\xaa\x15\xb7\x47\xf3\xc7\x2f\x64\xef\xe0\x83\xfd\xc8\xfa\x41\xdc\x4e\x8e\x1c\x95\xed\xe0\xd0\xa7\xed\xcb\xdf\xed\xe3\x42\x17\x77\x2f\x5b\x7e\x39\x84\xed\x8f\x63\xdd\xf4\xcd\xfc\x33\x65\x92\x12\x02\xdb\x3a\x4c\x9f\x88\x9f\x1d\x98\x1c\x3b\x7d\x68\x7c\xd2\xc8\x78\x53\xec\x8c\x4e\x53\x9a\x03\x0e\x5c\x5f\xaf\x8c\x3c\xe9\xdc\xd0\xdd\xdd\x58\x31\x68\x5b\xe0\x76\x61\x75\x65\x52\x5a\x98\x2e\x18\x17\x87\x51\xfa\xb3\xde\x28\x8b\x45\x29\x85\x5a\x41\x18\x02\x86\x70\x1b\x61\xbe\xf0\x68\xd7\x29\x73\x3c\x80\xfc\xa9\x72\x1a\x78\xef\xc0\xa1\xb1\xe3\x90\xa4\x60\x1d\xf9\x64\x68\x7e\x75\x37\xf1\x9d\x65\xc9\x67\x25\x77\x60\x70\x53\x17\x92\x6b\x65\x37\x72\x7e\x28\x58\xf2\x88\x5b\xf8\xa1\x78\x93\xa6\xed\x90\x2d\x0f\xd2\xee\xf3\xeb\xa5\x0d\xcd\x5e\x7b\x04\x67\x67\x65\x07\x9b\x13\x9e\xbe\xc3\x93\x3c\x1a\xc2\xc1\xf1\xdb\x29\x93\xa4\x95\x2f\x15\x63\x0c\x63\x5b\x53\x26\x0c\x61\xb8\x02\xcd\x88\xe4\x60\xc0\x70\x38\x36\x79\x5f\x23\x86\x77\xdf\x75\x17\xba\x20\x75\x59\xf6\x49\xe3\x1d\xf8\x6e\xdd\x76\x8f\xfa\xf8\x66\x2d\x77\xb0\xfd\xaa\xae\xfb\xb1\x66\xf1\x3d\x77\x9b\x01\xd7\x77\xdd\x8f\x93\xae\xea\x46\x6d\x77\xf0\xc1\xff\xa9\xf3\x0e\xc2\x6b\x30\x78\x77\x49\xe5\xfb\xee\xab\x92\xaa\xdf\x7a\x7f\x8b\xac\xfa\xe6\x8d\xf7\x6b\x36\x1d\x2e\x25\xf5\x8a\x80\x24\xa8\xbf\x23\x5b\x34\x3f\x72\xbd\x26\xd3\x66\xc3\xa2\x5e\x1b\x83\x73\x97\x13\xf2\x51\x73\x5c\x9e\x3c\x48\xf2\xfe\x8b\x67\x2e\xbf\x44\xfa\xbb\x8f\xb5\x71\x42\xbf\x3b\x7a\x68\xec\x96\x9a\xef\xc6\x4f\x32\x49\xec\x38\xa4\x5a\xda\x12\xd5\x9c\x7d\xcf\x92\x58\x24\x4a\x37\x4d\x8d\x48\xe2\x99\xe3\xfe\xcb\x8c\xea\x31\x76\x4e\x3c\xab\x9d\x77\x65\xd9\x1e\x7a\x35\x77\xdd\x1b\xb7\x83\xa5\x57\xbf\xe0\x02\x0f\x3b\xed\x7f\x4a\x38\x6f\x06\xb6\x03\x55\x2a\xf4\x9a\xba\xf7\x31\x2c\xf9\x52\x2f\x40\xf7\x3a\xe8\x15\xaa\xc7\x33\xaf\xee\xcb\x4a\x43\xf0\xbf\x00\x00\x00\xff\xff\x03\xe0\x9f\x12\x28\x23\x00\x00") +var _pkgUiStaticJsGraph_templateHandlebar = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5f\x8f\xe3\xb8\x0d\x7f\xef\xa7\x60\xd5\x97\x5d\x14\x9e\xdc\x5c\x71\xf7\x50\x38\x2e\xae\x7b\x83\x03\x0a\x0c\xb6\xd8\xdb\xbb\xd7\x80\xb1\xe8\x58\x1d\x59\xf2\x49\x72\xfe\x34\x98\xef\x5e\xc8\xb2\x3d\xce\x1f\x3b\xce\xcc\x34\xc0\xe6\x21\x9b\x91\x48\x8a\xa4\xc8\x1f\x69\x7a\xa1\xf9\xc4\x5c\xac\x41\xf0\x39\x5b\x19\x2c\xf3\xc5\xc6\x60\x59\x92\xd9\xef\x05\x7f\x7e\x66\x90\x4a\xb4\xf6\x68\x8f\x25\x7f\x82\xee\x13\x67\xda\x14\x2d\xd9\x1f\x15\x99\xdd\xc2\xaf\x1c\xd0\x34\x87\x34\x44\x7e\x3b\x32\x7a\x73\x44\x72\x48\x94\x6a\x19\xc9\x55\x74\xff\xdd\x09\x15\x40\xec\x68\xeb\xd0\x10\x82\xd1\x1b\x3b\x67\xf7\x0c\x4a\x89\x29\xe5\x5a\x72\x32\x73\xf6\xb0\x2d\x0d\x59\x2b\xb4\x82\x0f\xf5\x2f\xf8\x35\x17\x99\xfb\xeb\x83\x72\x64\x20\xd3\x06\x14\x6d\xa4\x50\x64\x3f\x32\x50\x58\xd0\x9c\xd1\xb6\x34\xac\xf6\x82\xff\x75\x64\x7c\xad\x70\xaa\x95\x33\x5a\x02\x75\xc2\x17\x42\x95\x95\x63\xc0\xd1\x61\x54\x1a\xbd\x16\x9c\xe6\xcc\xed\x4a\xc2\x9c\x90\x33\xc0\xca\xe9\x54\x17\xa5\x24\x47\x73\xa6\xb3\x8c\x81\x2d\x49\xca\x34\xa7\xf4\x69\xce\x32\x94\x96\x58\xb2\xdf\x7b\x91\xcf\xcf\xf1\xac\x35\xeb\xc4\x2f\x33\x2e\xd6\x13\x9c\xf5\xfd\x39\x5f\xf5\xc8\x68\x8d\x72\x61\x1d\x3a\x0b\x99\xd4\xe8\x22\x23\x56\xb9\x63\xc9\x59\xf9\x00\xb1\x28\x56\x60\x4d\x3a\x67\xfb\x3d\x94\xe8\xf2\x7f\x1b\xca\xc4\x16\x9e\x9f\x67\x5e\x88\x48\x67\xa2\x58\xcd\xf0\x3f\xb8\x8d\xa4\x46\x4e\xe6\x6e\x25\xb2\x7f\xac\xe7\xfb\x3d\x2c\x2b\x21\xf9\xef\x64\xea\x3b\xe8\x79\xd2\x96\x42\x29\x32\x0c\x50\xba\x39\xf3\xac\x8b\x76\x69\x82\xd1\xe7\x96\x8e\xe3\x4a\x28\x7f\xb1\xa7\xd2\xea\xbb\x6a\x29\x97\x4e\xc1\xd2\xa9\xa8\x34\xa2\x40\xb3\x03\xda\x52\x5a\x39\x5a\x2c\x9d\x62\xe0\x2f\x70\xce\x6c\xb5\x2c\x84\x63\xb0\x46\x59\x91\x0f\xa9\x9a\xa2\x0d\x97\x66\xf7\xe4\x14\x4b\x92\xd2\xee\x98\xb4\xb2\x4e\x17\x51\xb3\x38\x14\x45\x61\xbb\x95\x2c\x94\x25\xe3\x16\x05\x39\x23\xd2\x73\xf7\xa9\x4b\xe7\x9d\xda\xe8\xc5\x92\x08\x02\x0b\x04\x16\x40\x07\x69\x65\xac\x36\x10\xc5\xb3\x40\x7c\xea\xda\x70\xe6\xc9\xfa\xb2\x72\x4e\xab\xc6\x03\xe1\x0f\x76\xec\x33\x4e\x19\x56\xd2\x41\x40\x04\x5d\x3a\xef\x35\xe0\xc4\xab\xb2\xf6\xdf\xa9\xe7\x3b\x0c\x91\xbb\x32\x17\xa9\x56\x3e\xe2\x44\x12\x78\xa4\x48\xd1\xeb\x78\xa2\x62\x38\x7e\xe0\x1e\x83\x86\xb9\xe0\x9c\x54\xeb\xb9\x5a\x5c\x77\x61\xf7\xa7\x8a\xbc\xde\xba\x12\x8d\x13\x28\x17\x86\x6c\xa9\x95\xa5\x2b\x0d\x6d\xd8\xa1\x65\x7f\xbb\xad\xc7\x0a\x0d\x9a\x3d\x25\x65\xa6\x42\xf1\x45\x78\x31\x46\x1b\x40\xe9\x83\xb1\xfe\x8e\x38\xaa\x95\x4f\xed\x21\x80\xe9\x31\x6f\xd0\x28\xa1\x56\x07\xec\xcd\xda\x00\xff\x30\x40\x1c\xae\xfd\x39\x8a\x8e\x38\xbf\x7e\xfe\xf9\xf3\xdf\xe1\x93\x56\x6b\x7f\x96\xcb\x85\x05\xa7\xe1\x9f\x5a\x3b\xeb\x0c\x96\xa0\x70\xbd\x44\x73\x07\xf0\xd5\x6f\x19\xfa\xa3\x12\x86\x2c\xfc\x0b\xd7\x68\x53\x23\x4a\x77\x62\x89\xff\x18\xca\x0c\xd9\xfc\xee\x68\x33\x8a\x6e\xe4\x7e\x29\xac\x8b\x56\x46\xfb\x24\x30\x5a\xfa\x52\x84\xcb\x12\x15\xc9\x33\x8c\x00\x71\x25\x5b\x4e\x85\x6b\x6f\x73\xe4\x70\x69\x7b\xbc\x5e\xe0\x59\x56\x80\x58\x8a\x1e\x73\x24\x1c\x15\x2d\xa3\xc7\x35\x52\xae\x4e\x6a\x96\xc4\xd8\xa7\x93\x42\x3d\x31\xc8\x0d\x65\x73\xf6\x97\x3a\xc5\xda\x4a\x8b\x46\x60\x0b\x8e\x6d\xbb\xd1\xee\x75\x0a\x35\xa5\xd6\xe9\xd5\xaa\x5d\x49\x7e\xf1\x94\xf1\x0c\x93\x78\x26\xc5\xdb\x94\x6d\x89\x30\x75\x62\x4d\xa3\xba\xa7\x5a\x59\x2d\x69\x40\xfb\xa3\xdd\x51\xfd\x3f\x05\xda\x31\x0b\xe2\x59\x25\xcf\xae\xf7\x2e\xdf\xe1\xb2\x56\x80\xd4\xe0\x95\x79\xea\xa3\xb8\xe8\x73\xfb\x95\x06\xf6\xbc\x20\x14\x8a\x0c\x18\xf2\x95\x9d\xbd\xb4\x87\x8d\x4d\xe7\x8f\x38\x8a\x5b\x49\x68\x32\xb1\x1d\x24\x0e\xb9\x09\x0f\x5b\x67\x30\x75\xc4\x7d\x12\x66\xda\xa4\x5e\x0d\x5d\x95\xc4\xa1\x06\x40\x7b\x77\x92\x43\x43\x47\x96\x46\x17\xe4\x72\xaa\x6c\xe8\xcd\x16\xb5\x20\x30\x1e\x86\xc2\x0a\x94\x95\x94\x91\xa4\x6c\xc8\x4d\x8d\xd0\x80\xc8\x23\x14\x70\x5c\x39\xa4\xef\xa5\x7a\xe2\x47\x79\x0f\x6a\xd0\x28\x65\x5b\xdb\xd2\x45\x6d\xc5\x05\xb1\xc2\xf9\xfb\xfd\x35\x37\x42\x3d\x81\xcb\x09\x9c\x28\x28\xd8\x7f\x37\x6a\xf0\xd9\xf2\x05\xdd\xaf\xa8\x10\xaa\xb2\xa1\x9c\x8d\xb9\xad\xad\x64\x67\x40\xf7\xe5\x13\x45\x49\x28\x6d\x53\xdc\xdb\x39\x34\x04\xc3\xb8\xfd\x3e\x4c\x7b\x97\xdd\x04\xeb\x14\x97\x7d\xed\xfc\x04\x3a\x0b\x69\x30\xe5\x06\x7d\xc7\x3e\xe5\xfe\x7a\x4a\x8d\x93\x5b\xf1\x5f\x9a\xb3\xbf\x8d\x13\x35\xb5\x7e\xbf\xef\x89\xf5\x49\x79\xd1\xeb\x13\xa2\xfa\x6d\x71\x7d\x4d\x64\x43\xd7\xf1\x4e\x8a\xed\xee\xaa\x7e\x31\x7a\xf3\xae\xb1\x5d\xca\x6b\x42\x7b\x18\x88\xce\x34\x1f\x07\xfb\xb7\x01\xbb\x3e\xc0\xbd\x11\xe1\x6e\x1f\x0b\x1e\xe7\x48\xf1\x89\x91\xf0\x85\x36\x42\xf1\x3a\x16\xc8\xff\x2b\x8a\x37\x46\xc2\x12\xd3\xa7\x0d\x1a\x7e\x25\xd0\x0d\x11\x4e\x03\xba\x33\x50\xc7\xd1\xb5\xf5\x6a\x02\x66\x04\xdc\x23\xc5\xa7\xe0\x5d\xe7\xbc\x87\xc6\x63\x1d\xde\xc1\x87\xdf\xbe\x7e\xfa\x78\x89\xfb\x60\xd0\xf2\x9b\x72\x42\x5e\xe2\xa8\x7b\x1e\xdf\xe9\xa2\x9b\xb3\xdd\x6e\xb7\x8b\x1e\x1f\x23\x7e\xf9\x8e\x27\x02\x6c\x1b\x3a\xa4\xf8\x14\x80\x6d\x21\xf6\xfe\xc7\x4b\x74\x1d\xca\x92\xe2\x1d\xba\x7e\x9b\xf0\x3a\x3d\xa5\x7e\xe2\x6b\x54\x29\xbd\x63\x4e\x65\xda\x5c\x97\x52\xaf\x07\xd8\xeb\xc0\x71\xcc\xa0\xfe\xb0\xa8\x99\xf2\x75\x98\x63\xb5\xac\xea\x11\x8c\x50\x60\x29\xd5\x8a\xdb\xa3\xf9\xe3\x17\xb2\x77\xf0\xc1\x7e\x64\xfd\x20\x6e\x27\x47\x8e\xca\x76\x70\xe8\xd3\xf6\xe5\xef\xf6\x71\xa1\x8b\xbb\x97\x2d\xbf\x1c\xc2\xf6\xc7\xb1\x6e\xfa\x66\xfe\x99\x32\x49\x09\x81\x6d\x1d\xa6\x4f\xc4\xcf\x0e\x4c\x8e\x9d\x3e\x34\x3e\x69\x64\xbc\x29\x76\x46\xa7\x29\xcd\x01\x07\xae\xaf\x57\x46\x9e\x74\x6e\xe8\xee\x6e\xac\x18\xb4\x2d\x70\xbb\xb0\xba\x32\x29\x2d\x4c\x17\x8c\x8b\xc3\x28\xfd\x59\x6f\x94\xc5\xa2\x94\x42\xad\x20\x0c\x01\x43\xb8\x8d\x30\x5f\x78\xb4\xeb\x94\x39\x1c\x40\x62\xe5\x34\x4b\x7e\xaa\x9c\x06\xde\x3b\x74\x68\xf4\x78\x41\xda\x77\x96\x25\x9f\x95\xdc\x81\xc1\x4d\x5d\x38\x5e\x29\xe7\x87\x82\x25\x8f\xb8\x85\x1f\x8a\xf7\xd0\xea\x3e\x0f\xd2\xee\xf3\xeb\xa5\x0d\xcd\x5a\x7b\x04\x67\x67\x63\x07\x9b\x13\x9e\xb6\xc3\x93\x3b\x1a\xc2\xc1\x71\xdb\x29\x93\xa4\x95\x2f\x0d\x63\x0c\x63\x5b\x53\x26\x0a\x61\x98\x02\xcd\x48\xe4\x60\xa0\x70\x38\x26\xf9\xb6\x46\x0a\xdf\x7c\x97\x5d\xe8\x82\xd4\x65\xd9\x27\x8d\x76\xe0\xbb\x75\x9b\x3d\xea\xe3\x9b\xb5\xd8\xc1\xf6\xab\xba\xec\xc7\x9a\xc5\xf7\xd8\x6d\x06\x5c\xdf\x65\x3f\x4e\xba\xaa\x1b\xb5\xd9\xc1\x07\xff\xa7\x4e\x3b\x08\xaf\xc1\xe0\x9b\x4b\x2a\xdf\x67\x5f\x95\x54\xfd\x56\xfb\x3d\xb2\xea\xdd\x1b\xed\xd7\x6c\x3a\x5c\x4a\xea\x15\x01\x49\x50\x7f\x47\xb6\x68\x7e\xe4\x7a\x4d\xa6\xcd\x86\x45\xbd\x36\x06\xe7\x2e\x27\xe4\xa3\xe6\xb8\x3c\x79\x90\xe4\xfd\x17\xcf\x5c\x7e\x89\xf4\x77\x1f\x6b\xe3\x84\x7e\x77\xf4\xd0\xd8\x2d\x35\xdf\x8d\x9f\x64\x92\xd8\x71\x48\xb5\xb4\x25\xaa\x39\xfb\x9e\x25\xb1\x48\x94\x6e\x9a\x1a\x91\xc4\x33\xc7\xfd\x97\x19\xd5\x63\xec\x9c\x78\x56\x3b\xef\xca\xb2\x3d\xf4\x2a\xee\xba\x37\x6c\x07\x4b\xaf\x7e\xa1\x05\x1e\x76\xda\xff\x84\x70\xde\x0c\x6c\x07\xa8\x54\xe8\x35\x75\xef\x5f\x58\xf2\xa5\x5e\x80\xee\xf5\xcf\x2b\x54\x8f\x67\x5e\xdd\x97\x95\x86\xe0\x7f\x01\x00\x00\xff\xff\x4b\x0a\xff\x8a\x18\x23\x00\x00") func pkgUiStaticJsGraph_templateHandlebarBytes() ([]byte, error) { return bindataRead( @@ -540,7 +540,7 @@ func pkgUiStaticJsGraph_templateHandlebar() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/graph_template.handlebar", size: 9000, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/graph_template.handlebar", size: 8984, mode: os.FileMode(420), modTime: time.Unix(1570545416, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -560,7 +560,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapGridCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.css", size: 37644, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.css", size: 37644, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -580,7 +580,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapGridMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.min.css", size: 28977, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.min.css", size: 28977, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -600,7 +600,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapRebootCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.css", size: 4896, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.css", size: 4896, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -620,7 +620,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapRebootMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.min.css", size: 4019, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.min.css", size: 4019, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -640,7 +640,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap.min.css", size: 140936, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap.min.css", size: 140936, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -660,7 +660,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapBundleJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.js", size: 212345, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.js", size: 212345, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -680,7 +680,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapBundleMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.min.js", size: 70966, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.min.js", size: 70966, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -700,7 +700,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.min.js", size: 51039, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.min.js", size: 51039, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -720,7 +720,7 @@ func pkgUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs() (*asset, err return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 11273, mode: os.FileMode(420), modTime: time.Unix(1566355526, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 11273, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -740,7 +740,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsCssBootstrapGlyphiconsCss() (*asset, e return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.css", size: 14523, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.css", size: 14523, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -760,7 +760,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsCssBootstrapGlyphiconsMinCss() (*asset return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css", size: 11830, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css", size: 11830, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -780,7 +780,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Eot() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.eot", size: 98620, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.eot", size: 98620, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -800,7 +800,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Svg() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.svg", size: 507478, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.svg", size: 507478, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -820,7 +820,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Ttf() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.ttf", size: 98384, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.ttf", size: 98384, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -840,7 +840,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Woff() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff", size: 63712, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff", size: 63712, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -860,7 +860,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Woff2() (*a return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff2", size: 54420, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff2", size: 54420, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -880,7 +880,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Eot() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.eot", size: 31156, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.eot", size: 31156, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -900,7 +900,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Svg() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.svg", size: 107199, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.svg", size: 107199, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -920,7 +920,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Ttf() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.ttf", size: 30928, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.ttf", size: 30928, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -940,7 +940,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Woff() (*a return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff", size: 14712, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff", size: 14712, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -960,7 +960,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Woff2() (* return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff2", size: 12220, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff2", size: 12220, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -980,7 +980,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Eot() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.eot", size: 102152, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.eot", size: 102152, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1000,7 +1000,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Svg() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.svg", size: 378215, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.svg", size: 378215, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1020,7 +1020,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Ttf() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.ttf", size: 101932, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.ttf", size: 101932, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1040,7 +1040,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Woff() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff", size: 48704, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff", size: 48704, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1060,7 +1060,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Woff2() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff2", size: 38784, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff2", size: 38784, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1080,7 +1080,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.eot", size: 20127, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.eot", size: 20127, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1100,7 +1100,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.svg", size: 108738, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.svg", size: 108738, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1120,7 +1120,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.ttf", size: 45404, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.ttf", size: 45404, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1140,7 +1140,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff", size: 23424, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff", size: 23424, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1160,7 +1160,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff2", size: 18028, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff2", size: 18028, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1180,7 +1180,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeCss() (*asset return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.css", size: 51062, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.css", size: 51062, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1200,7 +1200,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeLess() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.less", size: 53867, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.less", size: 53867, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1220,7 +1220,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeMinCss() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.min.css", size: 42307, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.min.css", size: 42307, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1240,7 +1240,7 @@ func pkgUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinC return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1260,7 +1260,7 @@ func pkgUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinJ return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1280,7 +1280,7 @@ func pkgUiStaticVendorFuzzyFuzzyJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/fuzzy/fuzzy.js", size: 5669, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/fuzzy/fuzzy.js", size: 5669, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1300,7 +1300,7 @@ func pkgUiStaticVendorJsJquery331MinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery-3.3.1.min.js", size: 86927, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery-3.3.1.min.js", size: 86927, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1320,7 +1320,7 @@ func pkgUiStaticVendorJsJqueryHotkeysJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.hotkeys.js", size: 4490, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.hotkeys.js", size: 4490, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1340,7 +1340,7 @@ func pkgUiStaticVendorJsJqueryMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.min.js", size: 86671, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.min.js", size: 86671, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1360,7 +1360,7 @@ func pkgUiStaticVendorJsJquerySelectionJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.selection.js", size: 12881, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.selection.js", size: 12881, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1380,7 +1380,7 @@ func pkgUiStaticVendorJsPopperMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/popper.min.js", size: 19236, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/popper.min.js", size: 19236, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1400,7 +1400,7 @@ func pkgUiStaticVendorMomentMomentTimezoneWithDataMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment-timezone-with-data.min.js", size: 184495, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment-timezone-with-data.min.js", size: 184495, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1420,7 +1420,7 @@ func pkgUiStaticVendorMomentMomentMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment.min.js", size: 51825, mode: os.FileMode(420), modTime: time.Unix(1565577418, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment.min.js", size: 51825, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1440,7 +1440,7 @@ func pkgUiStaticVendorMustacheMustacheMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1460,7 +1460,7 @@ func pkgUiStaticVendorRickshawRickshawMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1480,7 +1480,7 @@ func pkgUiStaticVendorRickshawRickshawMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1500,7 +1500,7 @@ func pkgUiStaticVendorRickshawVendorD3LayoutMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1520,7 +1520,7 @@ func pkgUiStaticVendorRickshawVendorD3V3Js() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(420), modTime: time.Unix(1563780417, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/pkg/ui/static/js/graph.js b/pkg/ui/static/js/graph.js index 2722115f5c..b191477dcd 100644 --- a/pkg/ui/static/js/graph.js +++ b/pkg/ui/static/js/graph.js @@ -44,6 +44,9 @@ Prometheus.Graph.prototype.initialize = function() { if (self.options.tab === undefined) { self.options.tab = 1; } + if (self.options.max_source_resolution === undefined) { + self.options.max_source_resolution = "0s"; + } // Draw graph controls and container from Handlebars template. @@ -93,6 +96,8 @@ Prometheus.Graph.prototype.initialize = function() { self.stacked = self.queryForm.find("input[name=stacked]"); self.insertMetric = self.queryForm.find("select[name=insert_metric]"); self.refreshInterval = self.queryForm.find("select[name=refresh]"); + self.maxSourceResolutionInput = self.queryForm.find("select[name=max_source_resolution_input]"); + self.consoleTab = graphWrapper.find(".console"); self.graphTab = graphWrapper.find(".graph_container"); @@ -231,6 +236,8 @@ Prometheus.Graph.prototype.initialize = function() { stylePartialResponseBtn(); }); + self.maxSourceResolutionInput.val(self.options.max_source_resolution); + self.queryForm.submit(function() { self.consoleTab.addClass("reload"); self.graphTab.addClass("reload"); @@ -377,7 +384,6 @@ Prometheus.Graph.prototype.getOptions = function() { "range_input", "end_input", "step_input", - "downsample_input", "stacked", "moment_input" ]; @@ -390,6 +396,9 @@ Prometheus.Graph.prototype.getOptions = function() { } } }); + + options.max_source_resolution = self.maxSourceResolutionInput.val(); + options.expr = self.expr.val(); options.tab = self.options.tab; return options; @@ -521,7 +530,7 @@ Prometheus.Graph.prototype.submitQuery = function() { var startTime = new Date().getTime(); var rangeSeconds = self.parseDuration(self.rangeInput.val()); var resolution = parseInt(self.queryForm.find("input[name=step_input]").val()) || Math.max(Math.floor(rangeSeconds / 250), 1); - var maxSourceResolution = self.queryForm.find("select[name=max_source_resolution_input]").val(); + var maxSourceResolution = self.maxSourceResolutionInput.val() var endDate = self.getEndDate() / 1000; var moment = self.getMoment() / 1000; diff --git a/pkg/ui/static/js/graph_template.handlebar b/pkg/ui/static/js/graph_template.handlebar index 928ab3cdd2..937e488260 100644 --- a/pkg/ui/static/js/graph_template.handlebar +++ b/pkg/ui/static/js/graph_template.handlebar @@ -114,8 +114,8 @@

From 89576af892c5bc555baf2ae685c53a5209b7606d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2019 22:42:59 +0200 Subject: [PATCH 028/257] build(deps): bump github.com/prometheus/client_golang (#1662) Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.1.0 to 1.2.1. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/master/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.1.0...v1.2.1) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7409165622..62b62543c4 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/opentracing/basictracer-go v1.0.0 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 - github.com/prometheus/client_golang v1.1.0 + github.com/prometheus/client_golang v1.2.1 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 github.com/prometheus/common v0.6.0 github.com/prometheus/prometheus v1.8.2-0.20190913102521-8ab628b35467 // v1.8.2 is misleading as Prometheus does not have v2 module. diff --git a/go.sum b/go.sum index 312e68f3cf..f4383eaf65 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v0.0.0-20181017004759-096ff4a8a059/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -389,6 +390,8 @@ github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= +github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -399,6 +402,7 @@ github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -406,6 +410,7 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/prometheus v1.8.2-0.20190913102521-8ab628b35467 h1:B9IMa7s163/ZDSduepHHfOZZHSKdSbgo/bFY5c+FMAs= github.com/prometheus/prometheus v1.8.2-0.20190913102521-8ab628b35467/go.mod h1:aojjoH+vNHyJUTJoW15HoQWMKXxNhQylU6/G261nqxQ= From b7f3ac9e758d29765f8a0b676f898fef72d73e3a Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 29 Oct 2019 13:24:39 +0000 Subject: [PATCH 029/257] Added experimental filesystem bucket implementation (#1690) * Added experimental filesystem bucket implementation Usa cases: * See: https://github.com/observatorium/thanos-replicate/issues/7 * Local testing, demos Signed-off-by: Bartek Plotka * Fixed edge case. Signed-off-by: Bartek Plotka * Disabled one test case. We cannot rely on this. Signed-off-by: Bartek Plotka --- docs/storage.md | 20 +- go.sum | 1 + pkg/compact/compact_e2e_test.go | 1 + pkg/objstore/client/factory.go | 16 +- pkg/objstore/filesystem/filesystem.go | 210 ++++++++++++++++++ pkg/objstore/inmem/inmem.go | 10 +- .../objtesting/acceptance_e2e_test.go | 36 +++ pkg/objstore/objtesting/foreach.go | 18 ++ pkg/objstore/s3/s3.go | 4 + pkg/store/bucket_test.go | 2 + scripts/cfggen/main.go | 14 +- 11 files changed, 316 insertions(+), 16 deletions(-) create mode 100644 pkg/objstore/filesystem/filesystem.go diff --git a/docs/storage.md b/docs/storage.md index 27f4499af3..2dbf1245d7 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -57,10 +57,11 @@ Current object storage client implementations: |----------------------|-------------------|-----------|---------------| | [Google Cloud Storage](./storage.md#gcs) | Stable (production usage) | yes | @bwplotka | | [AWS/S3](./storage.md#s3) | Stable (production usage) | yes | @bwplotka | -| [Azure Storage Account](./storage.md#azure) | Stable (production usage) | yes | @vglafirov | +| [Azure Storage Account](./storage.md#azure) | Stable (production usage) | no | @vglafirov | | [OpenStack Swift](./storage.md#openstack-swift) | Beta (working PoCs, testing usage) | no | @sudhi-vm | | [Tencent COS](./storage.md#tencent-cos) | Beta (testing usage) | no | @jojohappy | | [AliYun OSS](./storage.md#aliyun-oss) | Beta (testing usage) | no | @shaulboozhiao,@wujinhu | +| [Local Filesystem](./storage.md#filesystem) | Beta (testing usage) | yes | @bwplotka | NOTE: Currently Thanos requires strong consistency (write-read) for object store implementation. @@ -354,3 +355,20 @@ config: ``` Use --objstore.config-file to reference to this configuration file. + +### Filesystem + +This storage type is used when user wants to store and access the bucket in the local filesystem. +We treat filesystem the same way we would treat object storage, so all optimization for remote bucket applies even though, +we might have the files locally. + +NOTE: This storage type is experimental and might be inefficient. It is NOT advised to use it as the main storage for metrics +in production environment. Particularly there is no planned support for distributed filesystems like NFS. +This is mainly useful for testing and demos. + +[embedmd]:# (flags/config_bucket_filesystem.txt yaml) +```yaml +type: FILESYSTEM +config: + directory: "" +``` diff --git a/go.sum b/go.sum index f4383eaf65..f57134141b 100644 --- a/go.sum +++ b/go.sum @@ -422,6 +422,7 @@ github.com/samuel/go-zookeeper v0.0.0-20190810000440-0ceca61e4d75 h1:cA+Ubq9qEVI github.com/samuel/go-zookeeper v0.0.0-20190810000440-0ceca61e4d75/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= +github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef h1:RoeI7K0oZIcUirMHsFpQjTVDrl1ouNh8T7v3eNsUxL0= github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 6888777411..7513a944e4 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -156,6 +156,7 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) { sort.Slice(rem, func(i, j int) bool { return rem[i].Compare(rem[j]) < 0 }) + // Only the level 3 block, the last source block in both resolutions should be left. testutil.Equals(t, []ulid.ULID{metas[9].ULID, m3.ULID, m4.ULID}, rem) diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 50caf4d8e7..145b2157a1 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -12,6 +12,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/cos" + "github.com/thanos-io/thanos/pkg/objstore/filesystem" "github.com/thanos-io/thanos/pkg/objstore/gcs" "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" @@ -22,12 +23,13 @@ import ( type ObjProvider string const ( - GCS ObjProvider = "GCS" - S3 ObjProvider = "S3" - AZURE ObjProvider = "AZURE" - SWIFT ObjProvider = "SWIFT" - COS ObjProvider = "COS" - ALIYUNOSS ObjProvider = "ALIYUNOSS" + FILESYSTEM ObjProvider = "FILESYSTEM" + GCS ObjProvider = "GCS" + S3 ObjProvider = "S3" + AZURE ObjProvider = "AZURE" + SWIFT ObjProvider = "SWIFT" + COS ObjProvider = "COS" + ALIYUNOSS ObjProvider = "ALIYUNOSS" ) type BucketConfig struct { @@ -63,6 +65,8 @@ func NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registe bucket, err = cos.NewBucket(logger, config, component) case string(ALIYUNOSS): bucket, err = oss.NewBucket(logger, config, component) + case string(FILESYSTEM): + bucket, err = filesystem.NewBucketFromConfig(config) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/pkg/objstore/filesystem/filesystem.go b/pkg/objstore/filesystem/filesystem.go new file mode 100644 index 0000000000..fc0a11a717 --- /dev/null +++ b/pkg/objstore/filesystem/filesystem.go @@ -0,0 +1,210 @@ +package filesystem + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/thanos-io/thanos/pkg/objstore" + "gopkg.in/yaml.v2" + + "github.com/thanos-io/thanos/pkg/runutil" + + "github.com/pkg/errors" +) + +// Config stores the configuration for storing and accessing blobs in filesystem. +type Config struct { + Directory string `yaml:"directory"` +} + +// Bucket implements the objstore.Bucket interfaces against filesystem that binary runs on. +// Methods from Bucket interface are thread-safe. Objects are assumed to be immutable. +// NOTE: It does not follow symbolic links. +type Bucket struct { + rootDir string +} + +// NewBucketFromConfig returns a new filesystem.Bucket from config. +func NewBucketFromConfig(conf []byte) (*Bucket, error) { + var c Config + if err := yaml.Unmarshal(conf, &c); err != nil { + return nil, err + } + if c.Directory == "" { + return nil, errors.New("missing directory for filesystem bucket") + } + return NewBucket(c.Directory) +} + +// NewBucket returns a new filesystem.Bucket. +func NewBucket(rootDir string) (*Bucket, error) { + absDir, err := filepath.Abs(rootDir) + if err != nil { + return nil, err + } + return &Bucket{rootDir: absDir}, nil +} + +// Iter calls f for each entry in the given directory. The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) error { + absDir := filepath.Join(b.rootDir, dir) + info, err := os.Stat(absDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.Wrapf(err, "stat %s", absDir) + } + if !info.IsDir() { + return nil + } + + files, err := ioutil.ReadDir(absDir) + if err != nil { + return err + } + for _, file := range files { + name := filepath.Join(dir, file.Name()) + + if file.IsDir() { + empty, err := isDirEmpty(filepath.Join(absDir, file.Name())) + if err != nil { + return err + } + + if empty { + // Skip empty directories. + continue + } + name += objstore.DirDelim + } + if err := f(name); err != nil { + return err + } + } + return nil +} + +// Get returns a reader for the given object name. +func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { + return b.GetRange(ctx, name, 0, -1) +} + +type rangeReaderCloser struct { + io.Reader + f *os.File +} + +func (r *rangeReaderCloser) Close() error { + return r.f.Close() +} + +// GetRange returns a new range reader for the given object name and range. +func (b *Bucket) GetRange(_ context.Context, name string, off, length int64) (io.ReadCloser, error) { + if name == "" { + return nil, errors.New("object name is empty") + } + + file := filepath.Join(b.rootDir, name) + if _, err := os.Stat(file); err != nil { + return nil, errors.Wrapf(err, "stat %s", file) + } + + f, err := os.OpenFile(file, os.O_RDONLY, 0666) + if err != nil { + return nil, err + } + + if off > 0 { + _, err := f.Seek(off, 0) + if err != nil { + return nil, errors.Wrapf(err, "seek %v", off) + } + } + + if length == -1 { + return f, nil + } + + return &rangeReaderCloser{Reader: io.LimitReader(f, length), f: f}, nil +} + +// Exists checks if the given directory exists in memory. +func (b *Bucket) Exists(_ context.Context, name string) (bool, error) { + info, err := os.Stat(filepath.Join(b.rootDir, name)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, errors.Wrapf(err, "stat %s", filepath.Join(b.rootDir, name)) + } + return !info.IsDir(), nil +} + +// Upload writes the file specified in src to into the memory. +func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (err error) { + file := filepath.Join(b.rootDir, name) + if err := os.MkdirAll(filepath.Dir(file), os.ModePerm); err != nil { + return err + } + + f, err := os.Create(file) + if err != nil { + return err + } + defer runutil.CloseWithErrCapture(&err, f, "close") + + if _, err := io.Copy(f, r); err != nil { + return errors.Wrapf(err, "copy to %s", file) + } + return nil +} + +func isDirEmpty(name string) (ok bool, err error) { + f, err := os.Open(name) + if err != nil { + return false, err + } + defer runutil.CloseWithErrCapture(&err, f, "dir open") + + if _, err = f.Readdir(1); err == io.EOF { + return true, nil + } + return false, err +} + +// Delete removes all data prefixed with the dir. +func (b *Bucket) Delete(_ context.Context, name string) error { + file := filepath.Join(b.rootDir, name) + for file != b.rootDir { + if err := os.RemoveAll(file); err != nil { + return errors.Wrapf(err, "rm %s", file) + } + file = filepath.Dir(file) + empty, err := isDirEmpty(file) + if err != nil { + return err + } + if !empty { + break + } + } + return nil +} + +// IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. +func (b *Bucket) IsObjNotFoundErr(err error) bool { + return os.IsNotExist(errors.Cause(err)) +} + +func (b *Bucket) Close() error { return nil } + +// Name returns the bucket name. +func (b *Bucket) Name() string { + return fmt.Sprintf("fs: %s", b.rootDir) +} diff --git a/pkg/objstore/inmem/inmem.go b/pkg/objstore/inmem/inmem.go index edddd97e08..25e3196b96 100644 --- a/pkg/objstore/inmem/inmem.go +++ b/pkg/objstore/inmem/inmem.go @@ -16,8 +16,8 @@ import ( var errNotFound = errors.New("inmem: object not found") -// Bucket implements the store.Bucket and shipper.Bucket interfaces against local memory. -// methods from Bucket interface are thread-safe. Object are assumed to be immutable. +// Bucket implements the objstore.Bucket interfaces against local memory. +// Methods from Bucket interface are thread-safe. Objects are assumed to be immutable. type Bucket struct { mtx sync.RWMutex objects map[string][]byte @@ -116,7 +116,11 @@ func (b *Bucket) GetRange(_ context.Context, name string, off, length int64) (io } if int64(len(file)) < off { - return nil, errors.Errorf("inmem: offset larger than content length. Len %d. Offset: %v", len(file), off) + return ioutil.NopCloser(bytes.NewReader(nil)), nil + } + + if length == -1 { + return ioutil.NopCloser(bytes.NewReader(file[off:])), nil } if int64(len(file)) <= off+length { diff --git a/pkg/objstore/objtesting/acceptance_e2e_test.go b/pkg/objstore/objtesting/acceptance_e2e_test.go index 54c5894ada..ad743bd60e 100644 --- a/pkg/objstore/objtesting/acceptance_e2e_test.go +++ b/pkg/objstore/objtesting/acceptance_e2e_test.go @@ -49,6 +49,29 @@ func TestObjStore_AcceptanceTest_e2e(t *testing.T) { testutil.Ok(t, err) testutil.Equals(t, "tes", string(content)) + // Unspecified range with offset. + rcUnspecifiedLen, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, -1) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, rcUnspecifiedLen.Close()) }() + content, err = ioutil.ReadAll(rcUnspecifiedLen) + testutil.Ok(t, err) + testutil.Equals(t, "test-data@", string(content)) + + // Out of band offset. Do not rely on outcome. + // NOTE: For various providers we have different outcome. + // * GCS is giving 416 status code + // * S3 errors immdiately with invalid range error. + // * inmem and filesystem are returning 0 bytes. + //rcOffset, err := bkt.GetRange(ctx, "id1/obj_1.some", 124141, 3) + + // Out of band length. We expect to read file fully. + rcLength, err := bkt.GetRange(ctx, "id1/obj_1.some", 3, 9999) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, rcLength.Close()) }() + content, err = ioutil.ReadAll(rcLength) + testutil.Ok(t, err) + testutil.Equals(t, "st-data@", string(content)) + ok, err = bkt.Exists(ctx, "id1/obj_1.some") testutil.Ok(t, err) testutil.Assert(t, ok, "expected exits") @@ -97,6 +120,7 @@ func TestObjStore_AcceptanceTest_e2e(t *testing.T) { })) testutil.Ok(t, bkt.Delete(ctx, "id1/obj_2.some")) + // Delete is expected to fail on non existing object. // NOTE: Don't rely on this. S3 is not complying with this as GCS is. // testutil.NotOk(t, bkt.Delete(ctx, "id1/obj_2.some")) @@ -108,5 +132,17 @@ func TestObjStore_AcceptanceTest_e2e(t *testing.T) { return nil })) testutil.Equals(t, []string{"id1/obj_1.some", "id1/obj_3.some"}, seen) + + testutil.Ok(t, bkt.Delete(ctx, "id2/obj_4.some")) + + seen = []string{} + testutil.Ok(t, bkt.Iter(ctx, "", func(fn string) error { + seen = append(seen, fn) + return nil + })) + expected = []string{"obj_5.some", "id1/"} + sort.Strings(expected) + sort.Strings(seen) + testutil.Equals(t, expected, seen) }) } diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go index 29f09b4258..40db25838e 100644 --- a/pkg/objstore/objtesting/foreach.go +++ b/pkg/objstore/objtesting/foreach.go @@ -1,9 +1,12 @@ package objtesting import ( + "io/ioutil" "os" "testing" + "github.com/thanos-io/thanos/pkg/objstore/filesystem" + "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/cos" @@ -30,6 +33,21 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) return } + // Mandatory Filesystem. + if ok := t.Run("filesystem", func(t *testing.T) { + t.Parallel() + + dir, err := ioutil.TempDir("", "filesystem-foreach-store-test") + testutil.Ok(t, err) + defer testutil.Ok(t, os.RemoveAll(dir)) + + b, err := filesystem.NewBucket(dir) + testutil.Ok(t, err) + testFn(t, b) + }); !ok { + return + } + // Optional GCS. if _, ok := os.LookupEnv("THANOS_SKIP_GCS_TESTS"); !ok { t.Run("gcs", func(t *testing.T) { diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 2c062463d5..7bbb96d585 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -251,6 +251,10 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( if err := opts.SetRange(off, off+length-1); err != nil { return nil, err } + } else if off > 0 { + if err := opts.SetRange(off, 0); err != nil { + return nil, err + } } r, err := b.client.GetObjectWithContext(ctx, b.name, name, *opts) if err != nil { diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 69ccb57cc0..7f2c2cebac 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -433,6 +433,8 @@ func TestBucketStore_Info(t *testing.T) { dir, err := ioutil.TempDir("", "bucketstore-test") testutil.Ok(t, err) + defer testutil.Ok(t, os.RemoveAll(dir)) + bucketStore, err := NewBucketStore( nil, nil, diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 7b79819415..ca9e1933f9 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -15,6 +15,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/cos" + "github.com/thanos-io/thanos/pkg/objstore/filesystem" "github.com/thanos-io/thanos/pkg/objstore/gcs" "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" @@ -29,12 +30,13 @@ import ( var ( bucketConfigs = map[client.ObjProvider]interface{}{ - client.AZURE: azure.Config{}, - client.GCS: gcs.Config{}, - client.S3: s3.DefaultConfig, - client.SWIFT: swift.SwiftConfig{}, - client.COS: cos.Config{}, - client.ALIYUNOSS: oss.Config{}, + client.AZURE: azure.Config{}, + client.GCS: gcs.Config{}, + client.S3: s3.DefaultConfig, + client.SWIFT: swift.SwiftConfig{}, + client.COS: cos.Config{}, + client.ALIYUNOSS: oss.Config{}, + client.FILESYSTEM: filesystem.Config{}, } tracingConfigs = map[trclient.TracingProvider]interface{}{ trclient.JAEGER: jaeger.Config{}, From 08b3dc6f6a772549d27f660f97dc6b54e98780ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Thu, 31 Oct 2019 18:51:29 +0100 Subject: [PATCH 030/257] cmd/thanos/bucket: expose metrics (#1702) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, the bucket web command generates and registers metrics but they are never actually exposed. This commit ensures that the metrics are exposed and leverages the recently created server package for consistency. This cleanup also helps prepare for the upcoming changes for https://github.com/thanos-io/thanos/issues/1657. Signed-off-by: Lucas Servén Marín --- cmd/thanos/bucket.go | 32 ++++++++++++++------------------ docs/components/bucket.md | 2 ++ pkg/ui/bucket.go | 8 ++++---- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 4bc7085e0b..e4bfc90802 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -4,23 +4,23 @@ import ( "context" "encoding/json" "fmt" - "net" - "net/http" "os" "sort" "strings" "text/template" "time" - "github.com/thanos-io/thanos/pkg/extflag" - "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact" + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/extflag" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/client" + "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/server" "github.com/thanos-io/thanos/pkg/ui" "github.com/thanos-io/thanos/pkg/verifier" @@ -32,7 +32,6 @@ import ( opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/route" "github.com/prometheus/prometheus/tsdb/labels" "golang.org/x/text/language" "golang.org/x/text/message" @@ -312,6 +311,7 @@ func registerBucketInspect(m map[string]setupFunc, root *kingpin.CmdClause, name func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) { cmd := root.Command("web", "Web interface for remote storage bucket") bind := cmd.Flag("listen", "HTTP host:port to listen on").Default("0.0.0.0:8080").String() + httpGracePeriod := regHTTPGracePeriodFlag(cmd) interval := cmd.Flag("refresh", "Refresh interval to download metadata from remote storage").Default("30m").Duration() timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() label := cmd.Flag("label", "Prometheus label to use as timeline title").String() @@ -319,9 +319,15 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str m[name+" web"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { ctx, cancel := context.WithCancel(context.Background()) - router := route.New() + statusProber := prober.NewProber(component.Bucket, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. + srv := server.NewHTTP(logger, reg, component.Bucket, statusProber, + server.WithListen(*bind), + server.WithGracePeriod(time.Duration(*httpGracePeriod)), + ) + bucketUI := ui.NewBucketUI(logger, *label) - bucketUI.Register(router, extpromhttp.NewInstrumentationMiddleware(reg)) + bucketUI.Register(srv, extpromhttp.NewInstrumentationMiddleware(reg)) if *interval < 5*time.Minute { level.Warn(logger).Log("msg", "Refreshing more often than 5m could lead to large data transfers") @@ -341,17 +347,7 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str cancel() }) - l, err := net.Listen("tcp", *bind) - if err != nil { - return errors.Wrapf(err, "listen HTTP on address %s", *bind) - } - - g.Add(func() error { - level.Info(logger).Log("msg", "Listening for query and metrics", "address", *bind) - return errors.Wrap(http.Serve(l, router), "serve web") - }, func(error) { - runutil.CloseWithLogOnErr(logger, l, "http listener") - }) + g.Add(srv.ListenAndServe, srv.Shutdown) return nil } diff --git a/docs/components/bucket.md b/docs/components/bucket.md index bf9247dca6..df6dcd284d 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -122,6 +122,8 @@ Flags: object store configuration. See format details: https://thanos.io/storage.md/#configuration --listen="0.0.0.0:8080" HTTP host:port to listen on + --http-grace-period=5s Time to wait after an interrupt received for HTTP + Server. --refresh=30m Refresh interval to download metadata from remote storage --timeout=5m Timeout to download metadata from remote storage diff --git a/pkg/ui/bucket.go b/pkg/ui/bucket.go index 8e023ba61b..84dfdbfdb5 100644 --- a/pkg/ui/bucket.go +++ b/pkg/ui/bucket.go @@ -6,8 +6,8 @@ import ( "time" "github.com/go-kit/kit/log" - "github.com/prometheus/common/route" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" + "github.com/thanos-io/thanos/pkg/server" ) // Bucket is a web UI representing state of buckets as a timeline. @@ -30,13 +30,13 @@ func NewBucketUI(logger log.Logger, label string) *Bucket { } // Register registers http routes for bucket UI. -func (b *Bucket) Register(r *route.Router, ins extpromhttp.InstrumentationMiddleware) { +func (b *Bucket) Register(s server.Server, ins extpromhttp.InstrumentationMiddleware) { instrf := func(name string, next func(w http.ResponseWriter, r *http.Request)) http.HandlerFunc { return ins.NewHandler(name, http.HandlerFunc(next)) } - r.Get("/", instrf("root", b.root)) - r.Get("/static/*filepath", instrf("static", b.serveStaticAsset)) + s.Handle("/", instrf("root", b.root)) + s.Handle("/static/*filepath", instrf("static", b.serveStaticAsset)) } // Handle / of bucket UIs. From 0724a8ae2b6fd61298288a17a21ecf6451b4fd33 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 1 Nov 2019 01:52:09 -0500 Subject: [PATCH 031/257] remove irrelevant questions on remote-receive.md (#1698) these questions are about merged PRs Signed-off-by: Brett Jones --- docs/proposals/201812_thanos-remote-receive.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/proposals/201812_thanos-remote-receive.md b/docs/proposals/201812_thanos-remote-receive.md index 34598f068f..accca60faa 100644 --- a/docs/proposals/201812_thanos-remote-receive.md +++ b/docs/proposals/201812_thanos-remote-receive.md @@ -201,9 +201,6 @@ Decisions of the design have consequences some of which will show themselves in * Bursting remote write API requests (after a rollout or easing of rate limiting), as Prometheuses may attempt to push their data simultaneously. - This could be solved with a combination of rate limiting and back-off on Prometheus’ side, plus alerts if replication lag gets too large due to rate limiting (or other factors). -* This proposal describes the write-ahead-log based remote write, which is not (yet) merged in Prometheus: https://github.com/prometheus/prometheus/pull/4588. This landing may impact durability characteristics. - - While there is work left the pull request seems to be close to completion -* For compaction to work as described in this proposal, vertical compaction in TSDB needs to be possible. Implemented but not merged yet: https://github.com/prometheus/tsdb/pull/370 * Additional safeguards may need to be put in place to ensure that hashring resizes do not occur on failed nodes, only once they have recovered and have successfully uploaded their blocks. [xxhash]: http://cyan4973.github.io/xxHash/ From 841075d051d1a07fa25d91b613eb80215e272154 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 1 Nov 2019 10:55:06 +0000 Subject: [PATCH 032/257] docs: Maintainers - Added associated company. (#1703) As agreed https://github.com/thanos-io/thanos/pull/1688#discussion_r340463729 Signed-off-by: Bartek Plotka --- MAINTAINERS.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 8803a0c0bd..a76fbe3b21 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,13 +1,13 @@ # Core Maintainers of this repository -| Name | Email | Slack | GitHub | -|-----------------------|------------------------|--------------------------|------------------------------------------------------------| -| Bartłomiej Płotka | bwplotka@gmail.com | `@bwplotka` | [@bwplotka](https://github.com/bwplotka) | -| Dominic Green | dom@improbable.io | `@domgreen` | [@domgreen](https://github.com/domgreen) | -| Frederic Branczyk | fbranczyk@gmail.com | `@brancz` | [@brancz](https://github.com/brancz) | -| Giedrius Statkevičius | giedriuswork@gmail.com | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS) | -| Povilas Versockas | p.versockas@gmail.com | `@povilasv` | [@povilasv](https://github.com/povilasv) | -| Matthias Loibl | mail@matthiasloibl.com | `@metalmatze` | [@metalmatze](https://github.com/metalmatze) | +| Name | Email | Slack | GitHub | Company | +|-----------------------|------------------------|--------------------------|---------------------------------------------|-------------------| +| Bartłomiej Płotka | bwplotka@gmail.com | `@bwplotka` | [@bwplotka](https://github.com/bwplotka) | Red Hat | +| Dominic Green | dom@improbable.io | `@domgreen` | [@domgreen](https://github.com/domgreen) | Improbable | +| Frederic Branczyk | fbranczyk@gmail.com | `@brancz` | [@brancz](https://github.com/brancz) | Red Hat | +| Giedrius Statkevičius | giedriuswork@gmail.com | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS) | AdForm | +| Povilas Versockas | p.versockas@gmail.com | `@povilasv` | [@povilasv](https://github.com/povilasv) | Utility Warehouse | +| Matthias Loibl | mail@matthiasloibl.com | `@metalmatze` | [@metalmatze](https://github.com/metalmatze)| Red Hat | We are bunch of people from different companies with various interests and skills. We are from different parts of Europe: Germany, Lithuania, Poland and UK. From c7e787d97bcf8aa2064816b85b0656ec0e55aaed Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 1 Nov 2019 11:20:06 +0000 Subject: [PATCH 033/257] Added official Governance page for Thanos. (#1688) * Added official Governance page for Thanos. All maintainers, please review and approve *only* if you agree. Signed-off-by: Bartek Plotka * Addressed comments. Signed-off-by: Bartek Plotka * Addressed commments (only FAQ update). Signed-off-by: Bartek Plotka --- docs/governance.md | 211 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 docs/governance.md diff --git a/docs/governance.md b/docs/governance.md new file mode 100644 index 0000000000..871d56c312 --- /dev/null +++ b/docs/governance.md @@ -0,0 +1,211 @@ + +--- +title: Governance +type: docs +menu: thanos +slug: /governance.md +--- + +# Governance + +This document describes the rules and governance of the project. It is a slightly modified version of the [Prometheus Governance](https://prometheus.io/governance/#governance-changes). + +It is meant to be followed by all the developers of the Thanos project and the Thanos community. Common terminology used in this governance document are listed below: + +* **Maintainers Team**: A core Thanos team that have owner access to http://github.com/thanos-io organization and all projects within it. Current list is available [here][maintainers-doc]. + +* **Triage Team**: Contributors who does not belong to Maintainer's team, but has `Triage` GitHub role on [Thanos](https://github.com/thanos-io/thanos) repository allowing to change GitHub issues and PRs statuses and labels. +They are listed [here](https://thanos.io/maintainers/#triage). + +* **The Thanos project**: The sum of all activities performed under the [thanos-io organization on GitHub][gh], concerning one or more repositories or the community. + +Both Triage and Maintainers have are part of [`thanos-io@googlegroups.com`][team] email list. + +## Values + +The Thanos developers and community are expected to follow the values defined in the [CNCF charter][charter], including the [CNCF Code of Conduct][coc]. +Furthermore, the Thanos community strives for kindness, giving feedback effectively, and building a welcoming environment. The Thanos developers generally decide by consensus and only resort to conflict resolution by a majority vote if consensus cannot be reached. + +## Decision making + +### Maintainers Team + +Team member status may be given to those who have made ongoing contributions to the Thanos project for at least 3 months. +This is usually in the form of code improvements and/or notable work on documentation, but organizing events or user support could also be taken into account. + +New members may be proposed by any existing Maintainer by email to [thanos-io][team]. It is highly desirable to reach consensus about acceptance of a new member. +However, the proposal is ultimately voted on by a formal [supermajority vote](#supermajority-vote) of Team Maintainers. + +If the new member proposal is accepted, the proposed team member should be contacted privately via email to confirm or deny their acceptance of team membership. +This email will also be CC'd to [thanos-io][team] for record-keeping purposes. + +If they choose to accept, the following steps are taken: + +* Maintainer is added to the [GitHub organization][gh] as _Owner_. +* Maintainer is added to the [thanos-io][team]. +* Maintainer is added to the list of team members [here][maintainers-doc] +* New maintainer is announced on the [Thanos Twitter](https://twitter.com/ThanosMetrics) by an existing team member. + +Team members may retire at any time by emailing [thanos-io@googlegroups.com][team]. + +Team members can be removed by [supermajority vote](#supermajority-vote) on [thanos-io@googlegroups.com][team]. For this vote, the member in question is not eligible to vote and does not count towards the quorum. + +Upon death of a member, their team membership ends automatically. + +### Triage Team + +Triage team has similar rules, however the contributions made to the projects does not need to be as significant as expected by potential maintainer. + +New members as well may be proposed by any existing Maintainer or Triage person by email to [thanos-io@googlegroups.com][team]. It is highly desirable to reach consensus about acceptance of a new member. +However, the proposal is ultimately voted on by a formal [majority vote](#majority-vote) (in comparison to Maintainer's vote which requires supermajority). + +If the new member proposal is accepted, the proposed team member should be contacted privately via email to confirm or deny their acceptance of team membership. +This email will also be CC'd to [thanos-io@googlegroups.com][team] for record-keeping purposes. + +If they choose to accept, the following steps are taken: + +* Triage member is added to the [Thanos project](http://github.com/thanos-io/thanos) with `Triage` access. +* Triage member is added to the [thanos-io][team]. +* Triage member is added to the list of Triage members [here][maintainers-doc]. +* New team Triage member are announced on the [Thanos Twitter][twitter] by an existing team member. + +Triage member may retire at any time by emailing [thanos-io@googlegroups.com][team]. It can be proposed to step up as Maintainer in any time as well. + +Triage member can be removed by [majority vote](#majority-vote) on [thanos-io@googlegroups.com][team]. For this vote, the member in question is not eligible to vote and does not count towards the quorum. +Only Maintainers team has right to vote. + +Upon death of a member, their Triage team membership ends automatically. + +### Technical decisions + +Smaller technical decisions are made informally and [lazy consensus](#consensus) is assumed. Technical decisions that span multiple parts of the Thanos project +should be discussed and made on the [GitHub issues][issues] and in most cases followed by proposal as described [here](https://thanos.io/contributing/#adding-new-features-components). + +Decisions are usually made by [lazy consensus](#consensus). If no consensus can be reached, the matter may be resolved by [majority vote](#majority-vote). + +### Governance changes + +Material changes to this document are discussed publicly on the [Thanos GitHub](http://github.com/thanos-io/thanos). +Any change requires a [supermajority](#supermajority-vote) in favor. Editorial changes may be made by [lazy consensus](#consensus) unless challenged. + +### Other matters + +Any matter that needs a decision, including but not limited to financial matters, may be called to a vote by any Maintainer if they deem it necessary. +For financial, private, or personnel matters, discussion and voting takes place on the [thanos-io@googlegroups.com][team]; Otherwise discussion and votes are held in public on the GitHub issues or #thanos-dev CNCF slack channel. + +## Voting + +The Thanos project usually runs by informal consensus, however sometimes a formal decision must be made. + +Depending on the subject matter, as laid out [above](#decision-making), different methods of voting are used. + +For all votes, voting must be open for at least one week. The end date should be clearly stated in the call to vote. +A vote may be called and closed early if enough votes have come in one way so that further votes cannot change the final decision. + +In all cases, all and only [Maintainers](#maintainers-team) are eligible to vote, with the sole exception of the forced removal of a team member, in which said member is not eligible to vote. + +Discussion and votes on personnel matters (including but not limited to team membership and maintainership) are held in private on the [thanos-io@googlegroups.com][team]. All other discussion and votes are held in public on the GitHub issues or #thanos-dev CNCF slack channel. + +For public discussions, anyone interested is encouraged to participate. Formal power to object or vote is limited to [Maintainers Team](#maintainers-team). + +### Governance + +It's important for the project to stay independent and focused on shared interest instead of single use case of one company or organization. + +We value open source values and freedom, that's why we limit Maintainers Team **votes to maximum two from a single organization or company.** + +We also encourage any other company interested in helping maintaining Thanos to join us to make sure we stay independent. + +### Consensus + +The default decision making mechanism for the Thanos project is [lazy consensus][lazy]. This means that any decision on technical issues is considered supported by the [team][team] as long as nobody objects. + +Silence on any consensus decision is implicit agreement and equivalent to explicit agreement. Explicit agreement may be stated at will. + +Consensus decisions can never override or go against the spirit of an earlier explicit vote. + +If any [member of Maintainers Team](#maintainers-team) raises objections, the team members work together towards a solution that all involved can accept. +This solution is again subject to lazy consensus. + +In case no consensus can be found, but a decision one way or the other must be made, anyone from [Maintainers Team](#maintainers-team) may call a formal [majority vote](#majority-vote). + +### Majority vote + +Majority votes must be called explicitly in a separate thread on the appropriate mailing list. The subject must be prefixed with `[VOTE]`. +In the body, the call to vote must state the proposal being voted on. It should reference any discussion leading up to this point. + +Votes may take the form of a single proposal, with the option to vote yes or no, or the form of multiple alternatives. + +A vote on a single proposal is considered successful if more vote in favor than against. + +If there are multiple alternatives, members may vote for one or more alternatives, or vote “no” to object to all alternatives. +It is not possible to cast an “abstain” vote. A vote on multiple alternatives is considered decided in favor of one alternative if it has received the most votes in favor, and a vote from more than half of those voting. Should no alternative reach this quorum, another vote on a reduced number of options may be called separately. + +### Supermajority vote + +Supermajority votes must be called explicitly in a separate thread on the appropriate mailing list. +The subject must be prefixed with `[VOTE]`. In the body, the call to vote must state the proposal being voted on. It should reference any discussion leading up to this point. + +Votes may take the form of a single proposal, with the option to vote yes or no, or the form of multiple alternatives. + +A vote on a single proposal is considered successful if at least two thirds of those eligible to vote vote in favor. + +If there are multiple alternatives, members may vote for one or more alternatives, or vote “no” to object to all alternatives. +A vote on multiple alternatives is considered decided in favor of one alternative if it has received the most votes in favor, and a vote from at least two thirds of those eligible to vote. Should no alternative reach this quorum, another vote on a reduced number of options may be called separately. + +## FAQ + +This section is informational. In case of disagreement, the rules above overrule any FAQ. + +### For majority vote, what if there is even number of maintainers and an equal amount of votes in favor than against? + +It has to be majority so the vote will be declined. + +### So what's the TLDR difference between majority vs supermajority? + +It's about number of up votes to agree on the decision. + +* majority: Majority of voters has to agree. +* supermajority: 2/3 voters has to agree. + +### How do I propose a decision? + +See [Contributor doc](https://thanos.io/contributing/#adding-new-features-components) + +### How do I become a team member? + +To become an official member of Maintainers Team, you should make ongoing contributions to one or more project(s) for at least three months. +At that point, a team member (typically a maintainer of the project) may propose you for membership. +The discussion about this will be held in private, and you will be informed privately when a decision has been made. A possible, but not required, graduation path is to become a maintainer first. + +Should the decision be in favor, your new membership will also be announced on the [Thanos Twitter][twitter] + +### How do I add a project? + +As a team member, propose the new project on the [Thanos GitHub Issue][issues]. However, currently to maintain project in our organization you have to become Thanos Maintainers. + +All are encourage to start their own project related to Thanos. Thanos team is happy to link to your poject in appriopriate page e.g https://thanos.io/integrations.md/ + +### How do I remove a Maintainer or Triage member? + +All members may resign by notifying the [thanos-io@googlegroups.com][team]. If you think a team member should be removed against their will, propose this to the [thanos-io@googlegroups.com][team]. +Discussions will be held there in private. + +### Can majority/supermajority vote be done on GitHub PR by just approving PR? + +No,`[VOTE]` email has to be created. + +### What if during majority/supermajority vote there is no answer after week? + +For majority voting this means that member that did not send a response agree with the proposal. + +For supermajority voting team has to wait for all answers. + +[twitter]: https://twitter.com/ThanosMetrics +[issues]: https://github.com/thanos-io/thanos/issues +[maintainers-doc]: https://thanos.io/maintainers/ +[team]: https://groups.google.com/forum/#!forum/thanos-io +[gh]: https://github.com/thanos-io +[charter]: https://www.cncf.io/about/charter/ +[coc]: https://github.com/cncf/foundation/blob/master/code-of-conduct.md +[lazy]: https://couchdb.apache.org/bylaws.html#lazy From 9d4d0bf408ed86eaab371c8f0c52d7bb5fe6e164 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Fri, 1 Nov 2019 18:16:20 +0100 Subject: [PATCH 034/257] .*: Introduce graceful shutdown for gRPC Servers (#1687) * Inroduce graceful shutdown for gRPC Signed-off-by: Kemal Akkoyun * Add missed cancel branch Signed-off-by: Kemal Akkoyun * Remove stutter from server structs Signed-off-by: Kemal Akkoyun * Close servers immediately if grace period is not specified Signed-off-by: Kemal Akkoyun * Update CHANGELOG Signed-off-by: Kemal Akkoyun * Rename TLS methods, clarify log messages Signed-off-by: Kemal Akkoyun * Document public functions Signed-off-by: Kemal Akkoyun * Fix review issues Signed-off-by: Kemal Akkoyun * Update bucket docs Signed-off-by: Kemal Akkoyun * trigger checks Signed-off-by: Kemal Akkoyun * trigger checks Signed-off-by: Kemal Akkoyun --- CHANGELOG.md | 1 + cmd/thanos/bucket.go | 29 +++--- cmd/thanos/compact.go | 14 ++- cmd/thanos/downsample.go | 14 ++- cmd/thanos/flags.go | 12 ++- cmd/thanos/main.go | 156 -------------------------------- cmd/thanos/query.go | 55 +++++------ cmd/thanos/receive.go | 53 ++++++----- cmd/thanos/rule.go | 55 +++++------ cmd/thanos/sidecar.go | 56 ++++++------ cmd/thanos/store.go | 69 +++++++------- docs/components/bucket.md | 4 +- docs/components/compact.md | 2 +- docs/components/query.md | 4 +- docs/components/rule.md | 4 +- docs/components/sidecar.md | 4 +- docs/components/store.md | 4 +- go.mod | 3 +- go.sum | 3 + pkg/server/grpc/grpc.go | 132 +++++++++++++++++++++++++++ pkg/server/grpc/option.go | 47 ++++++++++ pkg/server/{ => http}/http.go | 41 +++++---- pkg/server/{ => http}/option.go | 6 +- pkg/tls/options.go | 104 +++++++++++++++++++++ pkg/ui/bucket.go | 4 +- scripts/quickstart.sh | 93 ++++++++++--------- test/e2e/spinup_test.go | 5 + 27 files changed, 566 insertions(+), 408 deletions(-) create mode 100644 pkg/server/grpc/grpc.go create mode 100644 pkg/server/grpc/option.go rename pkg/server/{ => http}/http.go (64%) rename pkg/server/{ => http}/option.go (64%) create mode 100644 pkg/tls/options.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 059eddbcc2..68be8ef781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Added +- [#1687](https://github.com/thanos-io/thanos/pull/1687) Add a new `--grpc-grace-period` CLI option to components which serve gRPC to set how long to wait until gRPC Server shuts down. - [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. - [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss) for further information. - [#1680](https://github.com/thanos-io/thanos/pull/1680) Add a new `--http-grace-period` CLI option to components which serve HTTP to set how long to wait until HTTP Server shuts down. diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index e4bfc90802..244d8942a1 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -10,6 +10,15 @@ import ( "text/template" "time" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/oklog/run" + "github.com/oklog/ulid" + "github.com/olekukonko/tablewriter" + opentracing "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact" @@ -20,19 +29,9 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/runutil" - "github.com/thanos-io/thanos/pkg/server" + httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/ui" "github.com/thanos-io/thanos/pkg/verifier" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/oklog/run" - "github.com/oklog/ulid" - "github.com/olekukonko/tablewriter" - opentracing "github.com/opentracing/opentracing-go" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/prometheus/tsdb/labels" "golang.org/x/text/language" "golang.org/x/text/message" kingpin "gopkg.in/alecthomas/kingpin.v2" @@ -311,7 +310,7 @@ func registerBucketInspect(m map[string]setupFunc, root *kingpin.CmdClause, name func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) { cmd := root.Command("web", "Web interface for remote storage bucket") bind := cmd.Flag("listen", "HTTP host:port to listen on").Default("0.0.0.0:8080").String() - httpGracePeriod := regHTTPGracePeriodFlag(cmd) + _, httpGracePeriod := regHTTPFlags(cmd) interval := cmd.Flag("refresh", "Refresh interval to download metadata from remote storage").Default("30m").Duration() timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() label := cmd.Flag("label", "Prometheus label to use as timeline title").String() @@ -321,9 +320,9 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str statusProber := prober.NewProber(component.Bucket, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := server.NewHTTP(logger, reg, component.Bucket, statusProber, - server.WithListen(*bind), - server.WithGracePeriod(time.Duration(*httpGracePeriod)), + srv := httpserver.New(logger, reg, component.Bucket, statusProber, + httpserver.WithListen(*bind), + httpserver.WithGracePeriod(time.Duration(*httpGracePeriod)), ) bucketUI := ui.NewBucketUI(logger, *label) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 09ea8fba93..ba09994913 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -10,9 +10,6 @@ import ( "strings" "time" - "github.com/thanos-io/thanos/pkg/extflag" - "github.com/thanos-io/thanos/pkg/server" - "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -25,10 +22,12 @@ import ( "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/extflag" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/runutil" + httpserver "github.com/thanos-io/thanos/pkg/server/http" "gopkg.in/alecthomas/kingpin.v2" ) @@ -79,8 +78,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) { "Compaction index verification will ignore out of order label names."). Hidden().Default("false").Bool() - httpAddr := regHTTPAddrFlag(cmd) - httpGracePeriod := regHTTPGracePeriodFlag(cmd) + httpAddr, httpGracePeriod := regHTTPFlags(cmd) dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() @@ -179,9 +177,9 @@ func runCompact( statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := server.NewHTTP(logger, reg, component, statusProber, - server.WithListen(httpBindAddr), - server.WithGracePeriod(httpGracePeriod), + srv := httpserver.New(logger, reg, component, statusProber, + httpserver.WithListen(httpBindAddr), + httpserver.WithGracePeriod(httpGracePeriod), ) g.Add(srv.ListenAndServe, srv.Shutdown) diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 9e360cec67..655bc20b63 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -6,9 +6,6 @@ import ( "path/filepath" "time" - "github.com/thanos-io/thanos/pkg/extflag" - "github.com/thanos-io/thanos/pkg/server" - "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -23,10 +20,12 @@ import ( "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/extflag" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/runutil" + httpserver "github.com/thanos-io/thanos/pkg/server/http" kingpin "gopkg.in/alecthomas/kingpin.v2" ) @@ -34,8 +33,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application) { comp := component.Downsample cmd := app.Command(comp.String(), "continuously downsamples blocks in an object store bucket") - httpAddr := regHTTPAddrFlag(cmd) - httpGracePeriod := regHTTPGracePeriodFlag(cmd) + httpAddr, httpGracePeriod := regHTTPFlags(cmd) dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() @@ -126,9 +124,9 @@ func runDownsample( } // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := server.NewHTTP(logger, reg, comp, statusProber, - server.WithListen(httpBindAddr), - server.WithGracePeriod(httpGracePeriod), + srv := httpserver.New(logger, reg, comp, statusProber, + httpserver.WithListen(httpBindAddr), + httpserver.WithGracePeriod(httpGracePeriod), ) g.Add(srv.ListenAndServe, srv.Shutdown) diff --git a/cmd/thanos/flags.go b/cmd/thanos/flags.go index 135552e6f6..b6d7776086 100644 --- a/cmd/thanos/flags.go +++ b/cmd/thanos/flags.go @@ -19,29 +19,31 @@ func modelDuration(flags *kingpin.FlagClause) *model.Duration { func regGRPCFlags(cmd *kingpin.CmdClause) ( grpcBindAddr *string, + grpcGracePeriod *model.Duration, grpcTLSSrvCert *string, grpcTLSSrvKey *string, grpcTLSSrvClientCA *string, ) { grpcBindAddr = cmd.Flag("grpc-address", "Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable from other components."). Default("0.0.0.0:10901").String() + grpcGracePeriod = modelDuration(cmd.Flag("grpc-grace-period", "Time to wait after an interrupt received for GRPC Server.").Default("2m")) // by default it's the same as query.timeout. grpcTLSSrvCert = cmd.Flag("grpc-server-tls-cert", "TLS Certificate for gRPC server, leave blank to disable TLS").Default("").String() grpcTLSSrvKey = cmd.Flag("grpc-server-tls-key", "TLS Key for the gRPC server, leave blank to disable TLS").Default("").String() grpcTLSSrvClientCA = cmd.Flag("grpc-server-tls-client-ca", "TLS CA to verify clients against. If no client CA is specified, there is no client verification on server side. (tls.NoClientCert)").Default("").String() return grpcBindAddr, + grpcGracePeriod, grpcTLSSrvCert, grpcTLSSrvKey, grpcTLSSrvClientCA } -func regHTTPAddrFlag(cmd *kingpin.CmdClause) *string { - return cmd.Flag("http-address", "Listen host:port for HTTP endpoints.").Default("0.0.0.0:10902").String() -} +func regHTTPFlags(cmd *kingpin.CmdClause) (httpBindAddr *string, httpGracePeriod *model.Duration) { + httpBindAddr = cmd.Flag("http-address", "Listen host:port for HTTP endpoints.").Default("0.0.0.0:10902").String() + httpGracePeriod = modelDuration(cmd.Flag("http-grace-period", "Time to wait after an interrupt received for HTTP Server.").Default("2m")) // by default it's the same as query.timeout. -func regHTTPGracePeriodFlag(cmd *kingpin.CmdClause) *model.Duration { - return modelDuration(cmd.Flag("http-grace-period", "Time to wait after an interrupt received for HTTP Server.").Default("5s")) + return httpBindAddr, httpGracePeriod } func regCommonObjStoreFlags(cmd *kingpin.CmdClause, suffix string, required bool, extraDesc ...string) *extflag.PathOrContent { diff --git a/cmd/thanos/main.go b/cmd/thanos/main.go index 3e93628e9b..e4b7a06e72 100644 --- a/cmd/thanos/main.go +++ b/cmd/thanos/main.go @@ -2,39 +2,25 @@ package main import ( "context" - "crypto/tls" - "crypto/x509" "fmt" "io" - "io/ioutil" - "math" "os" "os/signal" "path/filepath" "runtime" - "runtime/debug" "syscall" gmetrics "github.com/armon/go-metrics" gprom "github.com/armon/go-metrics/prometheus" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/oklog/run" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" - "github.com/thanos-io/thanos/pkg/store/storepb" - "github.com/thanos-io/thanos/pkg/tracing" "github.com/thanos-io/thanos/pkg/tracing/client" "go.uber.org/automaxprocs/maxprocs" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/status" "gopkg.in/alecthomas/kingpin.v2" ) @@ -222,145 +208,3 @@ func interrupt(logger log.Logger, cancel <-chan struct{}) error { return errors.New("canceled") } } - -func defaultGRPCTLSServerOpts(logger log.Logger, cert, key, clientCA string) ([]grpc.ServerOption, error) { - opts := []grpc.ServerOption{} - tlsCfg, err := defaultTLSServerOpts(log.With(logger, "protocol", "gRPC"), cert, key, clientCA) - if err != nil { - return opts, err - } - if tlsCfg != nil { - opts = append(opts, grpc.Creds(credentials.NewTLS(tlsCfg))) - } - return opts, nil -} - -func defaultTLSServerOpts(logger log.Logger, cert, key, clientCA string) (*tls.Config, error) { - if key == "" && cert == "" { - if clientCA != "" { - return nil, errors.New("when a client CA is used a server key and certificate must also be provided") - } - - level.Info(logger).Log("msg", "disabled TLS, key and cert must be set to enable") - return nil, nil - } - - level.Info(logger).Log("msg", "enabling server side TLS") - - if key == "" || cert == "" { - return nil, errors.New("both server key and certificate must be provided") - } - - tlsCfg := &tls.Config{ - MinVersion: tls.VersionTLS12, - } - - tlsCert, err := tls.LoadX509KeyPair(cert, key) - if err != nil { - return nil, errors.Wrap(err, "server credentials") - } - - tlsCfg.Certificates = []tls.Certificate{tlsCert} - - if clientCA != "" { - caPEM, err := ioutil.ReadFile(clientCA) - if err != nil { - return nil, errors.Wrap(err, "reading client CA") - } - - certPool := x509.NewCertPool() - if !certPool.AppendCertsFromPEM(caPEM) { - return nil, errors.Wrap(err, "building client CA") - } - tlsCfg.ClientCAs = certPool - tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert - - level.Info(logger).Log("msg", "server TLS client verification enabled") - } - - return tlsCfg, nil -} - -func defaultTLSClientOpts(logger log.Logger, cert, key, caCert, serverName string) (*tls.Config, error) { - var certPool *x509.CertPool - if caCert != "" { - caPEM, err := ioutil.ReadFile(caCert) - if err != nil { - return nil, errors.Wrap(err, "reading client CA") - } - - certPool = x509.NewCertPool() - if !certPool.AppendCertsFromPEM(caPEM) { - return nil, errors.Wrap(err, "building client CA") - } - level.Info(logger).Log("msg", "TLS client using provided certificate pool") - } else { - var err error - certPool, err = x509.SystemCertPool() - if err != nil { - return nil, errors.Wrap(err, "reading system certificate pool") - } - level.Info(logger).Log("msg", "TLS client using system certificate pool") - } - - tlsCfg := &tls.Config{ - RootCAs: certPool, - } - - if serverName != "" { - tlsCfg.ServerName = serverName - } - - if (key != "") != (cert != "") { - return nil, errors.New("both client key and certificate must be provided") - } - - if cert != "" { - cert, err := tls.LoadX509KeyPair(cert, key) - if err != nil { - return nil, errors.Wrap(err, "client credentials") - } - tlsCfg.Certificates = []tls.Certificate{cert} - level.Info(logger).Log("msg", "TLS client authentication enabled") - } - return tlsCfg, nil -} - -func newStoreGRPCServer(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, srv storepb.StoreServer, opts []grpc.ServerOption) *grpc.Server { - met := grpc_prometheus.NewServerMetrics() - met.EnableHandlingTimeHistogram( - grpc_prometheus.WithHistogramBuckets([]float64{ - 0.001, 0.01, 0.05, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, - }), - ) - panicsTotal := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "thanos_grpc_req_panics_recovered_total", - Help: "Total number of gRPC requests recovered from internal panic.", - }) - reg.MustRegister(met, panicsTotal) - - grpcPanicRecoveryHandler := func(p interface{}) (err error) { - panicsTotal.Inc() - level.Error(logger).Log("msg", "recovered from panic", "panic", p, "stack", debug.Stack()) - return status.Errorf(codes.Internal, "%s", p) - } - opts = append(opts, - grpc.MaxSendMsgSize(math.MaxInt32), - grpc_middleware.WithUnaryServerChain( - met.UnaryServerInterceptor(), - tracing.UnaryServerInterceptor(tracer), - grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), - ), - grpc_middleware.WithStreamServerChain( - met.StreamServerInterceptor(), - tracing.StreamServerInterceptor(tracer), - grpc_recovery.StreamServerInterceptor(grpc_recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), - ), - ) - - s := grpc.NewServer(opts...) - storepb.RegisterStoreServer(s, srv) - met.InitializeMetrics(s) - - return s -} diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index c9d93b4feb..0be1e769b0 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math" - "net" "net/http" "path" "time" @@ -31,8 +30,10 @@ import ( "github.com/thanos-io/thanos/pkg/query" v1 "github.com/thanos-io/thanos/pkg/query/api" "github.com/thanos-io/thanos/pkg/runutil" - "github.com/thanos-io/thanos/pkg/server" + grpcserver "github.com/thanos-io/thanos/pkg/server/grpc" + httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/store" + "github.com/thanos-io/thanos/pkg/tls" "github.com/thanos-io/thanos/pkg/tracing" "github.com/thanos-io/thanos/pkg/ui" "google.golang.org/grpc" @@ -45,9 +46,8 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { comp := component.Query cmd := app.Command(comp.String(), "query node exposing PromQL enabled Query API with data retrieved from multiple store nodes") - httpBindAddr := regHTTPAddrFlag(cmd) - httpGracePeriod := regHTTPGracePeriodFlag(cmd) - grpcBindAddr, srvCert, srvKey, srvClientCA := regGRPCFlags(cmd) + httpBindAddr, httpGracePeriod := regHTTPFlags(cmd) + grpcBindAddr, grpcGracePeriod, grpcCert, grpcKey, grpcClientCA := regGRPCFlags(cmd) secure := cmd.Flag("grpc-client-tls-secure", "Use TLS when talking to the gRPC server").Default("false").Bool() cert := cmd.Flag("grpc-client-tls-cert", "TLS Certificates to use to identify this client to the server").Default("").String() @@ -133,9 +133,10 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { reg, tracer, *grpcBindAddr, - *srvCert, - *srvKey, - *srvClientCA, + time.Duration(*grpcGracePeriod), + *grpcCert, + *grpcKey, + *grpcClientCA, *secure, *cert, *key, @@ -201,7 +202,7 @@ func storeClientGRPCOpts(logger log.Logger, reg *prometheus.Registry, tracer ope level.Info(logger).Log("msg", "enabling client to server TLS") - tlsCfg, err := defaultTLSClientOpts(logger, cert, key, caCert, serverName) + tlsCfg, err := tls.NewClientConfig(logger, cert, key, caCert, serverName) if err != nil { return nil, err } @@ -216,9 +217,10 @@ func runQuery( reg *prometheus.Registry, tracer opentracing.Tracer, grpcBindAddr string, - srvCert string, - srvKey string, - srvClientCA string, + grpcGracePeriod time.Duration, + grpcCert string, + grpcKey string, + grpcClientCA string, secure bool, cert string, key string, @@ -380,9 +382,9 @@ func runQuery( api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := server.NewHTTP(logger, reg, comp, statusProber, - server.WithListen(httpBindAddr), - server.WithGracePeriod(httpGracePeriod), + srv := httpserver.New(logger, reg, comp, statusProber, + httpserver.WithListen(httpBindAddr), + httpserver.WithGracePeriod(httpGracePeriod), ) srv.Handle("/", router) @@ -390,24 +392,23 @@ func runQuery( } // Start query (proxy) gRPC StoreAPI. { - l, err := net.Listen("tcp", grpcBindAddr) + tlsCfg, err := tls.NewServerConfig(log.With(logger, "protocol", "gRPC"), grpcCert, grpcKey, grpcClientCA) if err != nil { - return errors.Wrap(err, "listen gRPC on address") + return errors.Wrap(err, "setup gRPC server") } - logger := log.With(logger, "component", component.Query.String()) - opts, err := defaultGRPCTLSServerOpts(logger, srvCert, srvKey, srvClientCA) - if err != nil { - return errors.Wrap(err, "build gRPC server") - } - s := newStoreGRPCServer(logger, reg, tracer, proxy, opts) + s := grpcserver.New(logger, reg, tracer, comp, proxy, + grpcserver.WithListen(grpcBindAddr), + grpcserver.WithGracePeriod(grpcGracePeriod), + grpcserver.WithTLSConfig(tlsCfg), + ) g.Add(func() error { - level.Info(logger).Log("msg", "Listening for StoreAPI gRPC", "address", grpcBindAddr) statusProber.SetReady() - return errors.Wrap(s.Serve(l), "serve gRPC") - }, func(error) { - s.Stop() + return s.ListenAndServe() + }, func(err error) { + statusProber.SetNotReady(err) + s.Shutdown(err) }) } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 649b03ef5c..1424a13506 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -3,14 +3,10 @@ package main import ( "context" "fmt" - "net" "os" "strings" "time" - "github.com/thanos-io/thanos/pkg/extflag" - "github.com/thanos-io/thanos/pkg/server" - "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -22,13 +18,16 @@ import ( "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/extflag" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/receive" "github.com/thanos-io/thanos/pkg/runutil" + grpcserver "github.com/thanos-io/thanos/pkg/server/grpc" + httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/shipper" "github.com/thanos-io/thanos/pkg/store" - "google.golang.org/grpc" + "github.com/thanos-io/thanos/pkg/tls" kingpin "gopkg.in/alecthomas/kingpin.v2" ) @@ -36,9 +35,8 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { comp := component.Receive cmd := app.Command(comp.String(), "Accept Prometheus remote write API requests and write to local tsdb (EXPERIMENTAL, this may change drastically without notice)") - httpBindAddr := regHTTPAddrFlag(cmd) - httpGracePeriod := regHTTPGracePeriodFlag(cmd) - grpcBindAddr, grpcCert, grpcKey, grpcClientCA := regGRPCFlags(cmd) + httpBindAddr, httpGracePeriod := regHTTPFlags(cmd) + grpcBindAddr, grpcGracePeriod, grpcCert, grpcKey, grpcClientCA := regGRPCFlags(cmd) rwAddress := cmd.Flag("remote-write.address", "Address to listen on for remote write requests."). Default("0.0.0.0:19291").String() @@ -107,6 +105,7 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { reg, tracer, *grpcBindAddr, + time.Duration(*grpcGracePeriod), *grpcCert, *grpcKey, *grpcClientCA, @@ -141,6 +140,7 @@ func runReceive( reg *prometheus.Registry, tracer opentracing.Tracer, grpcBindAddr string, + grpcGracePeriod time.Duration, grpcCert string, grpcKey string, grpcClientCA string, @@ -178,11 +178,11 @@ func runReceive( } localStorage := &tsdb.ReadyStorage{} - rwTLSConfig, err := defaultTLSServerOpts(log.With(logger, "protocol", "HTTP"), rwServerCert, rwServerKey, rwServerClientCA) + rwTLSConfig, err := tls.NewServerConfig(log.With(logger, "protocol", "HTTP"), rwServerCert, rwServerKey, rwServerClientCA) if err != nil { return err } - rwTLSClientConfig, err := defaultTLSClientOpts(logger, rwClientCert, rwClientKey, rwClientServerCA, rwClientServerName) + rwTLSClientConfig, err := tls.NewClientConfig(logger, rwClientCert, rwClientKey, rwClientServerCA, rwClientServerName) if err != nil { return err } @@ -339,42 +339,41 @@ func runReceive( level.Debug(logger).Log("msg", "setting up http server") // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := server.NewHTTP(logger, reg, comp, statusProber, - server.WithListen(httpBindAddr), - server.WithGracePeriod(httpGracePeriod), + srv := httpserver.New(logger, reg, comp, statusProber, + httpserver.WithListen(httpBindAddr), + httpserver.WithGracePeriod(httpGracePeriod), ) g.Add(srv.ListenAndServe, srv.Shutdown) level.Debug(logger).Log("msg", "setting up grpc server") { - var ( - s *grpc.Server - l net.Listener - ) + var s *grpcserver.Server startGRPC := make(chan struct{}) g.Add(func() error { defer close(startGRPC) - opts, err := defaultGRPCTLSServerOpts(logger, grpcCert, grpcKey, grpcClientCA) + + tlsCfg, err := tls.NewServerConfig(log.With(logger, "protocol", "gRPC"), grpcCert, grpcKey, grpcClientCA) if err != nil { return errors.Wrap(err, "setup gRPC server") } for range dbReady { if s != nil { - s.Stop() - } - l, err = net.Listen("tcp", grpcBindAddr) - if err != nil { - return errors.Wrap(err, "listen API address") + s.Shutdown(errors.New("reload hashrings")) } tsdbStore := store.NewTSDBStore(log.With(logger, "component", "thanos-tsdb-store"), nil, localStorage.Get(), component.Receive, lset) - s = newStoreGRPCServer(logger, &receive.UnRegisterer{Registerer: reg}, tracer, tsdbStore, opts) + + s = grpcserver.New(logger, &receive.UnRegisterer{Registerer: reg}, tracer, comp, tsdbStore, + grpcserver.WithListen(grpcBindAddr), + grpcserver.WithGracePeriod(grpcGracePeriod), + grpcserver.WithTLSConfig(tlsCfg), + ) startGRPC <- struct{}{} } return nil - }, func(error) { + }, func(err error) { if s != nil { - s.Stop() + s.Shutdown(err) } }) // We need to be able to start and stop the gRPC server @@ -382,7 +381,7 @@ func runReceive( g.Add(func() error { for range startGRPC { level.Info(logger).Log("msg", "listening for StoreAPI gRPC", "address", grpcBindAddr) - if err := s.Serve(l); err != nil { + if err := s.ListenAndServe(); err != nil { return errors.Wrap(err, "serve gRPC") } } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index dedb979b2c..3d0345b2b0 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -17,9 +17,6 @@ import ( "syscall" "time" - "github.com/thanos-io/thanos/pkg/extflag" - "github.com/thanos-io/thanos/pkg/server" - "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -41,6 +38,7 @@ import ( "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/discovery/cache" "github.com/thanos-io/thanos/pkg/discovery/dns" + "github.com/thanos-io/thanos/pkg/extflag" "github.com/thanos-io/thanos/pkg/extprom" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" "github.com/thanos-io/thanos/pkg/objstore/client" @@ -49,9 +47,12 @@ import ( thanosrule "github.com/thanos-io/thanos/pkg/rule" v1 "github.com/thanos-io/thanos/pkg/rule/api" "github.com/thanos-io/thanos/pkg/runutil" + grpcserver "github.com/thanos-io/thanos/pkg/server/grpc" + httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/shipper" "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/tls" "github.com/thanos-io/thanos/pkg/tracing" "github.com/thanos-io/thanos/pkg/ui" "gopkg.in/alecthomas/kingpin.v2" @@ -62,9 +63,8 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { comp := component.Rule cmd := app.Command(comp.String(), "ruler evaluating Prometheus rules against given Query nodes, exposing Store API and storing old blocks in bucket") - httpBindAddr := regHTTPAddrFlag(cmd) - httpGracePeriod := regHTTPGracePeriodFlag(cmd) - grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) + httpBindAddr, httpGracePeriod := regHTTPFlags(cmd) + grpcBindAddr, grpcGracePeriod, grpcCert, grpcKey, grpcClientCA := regGRPCFlags(cmd) labelStrs := cmd.Flag("label", "Labels to be applied to all generated metrics (repeated). Similar to external labels for Prometheus, used to identify ruler and its blocks as unique source."). PlaceHolder("=\"\"").Strings() @@ -159,9 +159,10 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { *alertmgrs, *alertmgrsTimeout, *grpcBindAddr, - *cert, - *key, - *clientCA, + time.Duration(*grpcGracePeriod), + *grpcCert, + *grpcKey, + *grpcClientCA, *httpBindAddr, time.Duration(*httpGracePeriod), *webRoutePrefix, @@ -195,9 +196,10 @@ func runRule( alertmgrURLs []string, alertmgrsTimeout time.Duration, grpcBindAddr string, - cert string, - key string, - clientCA string, + grpcGracePeriod time.Duration, + grpcCert string, + grpcKey string, + grpcClientCA string, httpBindAddr string, httpGracePeriod time.Duration, webRoutePrefix string, @@ -502,25 +504,24 @@ func runRule( statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Start gRPC server. { - l, err := net.Listen("tcp", grpcBindAddr) - if err != nil { - return errors.Wrap(err, "listen API address") - } - logger := log.With(logger, "component", component.Rule.String()) - store := store.NewTSDBStore(logger, reg, db, component.Rule, lset) - opts, err := defaultGRPCTLSServerOpts(logger, cert, key, clientCA) + tlsCfg, err := tls.NewServerConfig(log.With(logger, "protocol", "gRPC"), grpcCert, grpcKey, grpcClientCA) if err != nil { - return errors.Wrap(err, "setup gRPC options") + return errors.Wrap(err, "setup gRPC server") } - s := newStoreGRPCServer(logger, reg, tracer, store, opts) + s := grpcserver.New(logger, reg, tracer, comp, store, + grpcserver.WithListen(grpcBindAddr), + grpcserver.WithGracePeriod(grpcGracePeriod), + grpcserver.WithTLSConfig(tlsCfg), + ) g.Add(func() error { statusProber.SetReady() - return errors.Wrap(s.Serve(l), "serve gRPC") - }, func(error) { - s.Stop() + return s.ListenAndServe() + }, func(err error) { + statusProber.SetNotReady(err) + s.Shutdown(err) }) } // Start UI & metrics HTTP server. @@ -552,9 +553,9 @@ func runRule( api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := server.NewHTTP(logger, reg, comp, statusProber, - server.WithListen(httpBindAddr), - server.WithGracePeriod(httpGracePeriod), + srv := httpserver.New(logger, reg, comp, statusProber, + httpserver.WithListen(httpBindAddr), + httpserver.WithGracePeriod(httpGracePeriod), ) srv.Handle("/", router) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index e9668065a6..437e729f69 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -3,14 +3,10 @@ package main import ( "context" "math" - "net" "net/url" "sync" "time" - "github.com/thanos-io/thanos/pkg/extflag" - "github.com/thanos-io/thanos/pkg/server" - "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -21,24 +17,27 @@ import ( "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/extflag" thanosmodel "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/reloader" "github.com/thanos-io/thanos/pkg/runutil" + grpcserver "github.com/thanos-io/thanos/pkg/server/grpc" + httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/shipper" "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/tls" "gopkg.in/alecthomas/kingpin.v2" ) func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(component.Sidecar.String(), "sidecar for Prometheus server") - httpBindAddr := regHTTPAddrFlag(cmd) - httpGracePeriod := regHTTPGracePeriodFlag(cmd) - grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) + httpBindAddr, httpGracePeriod := regHTTPFlags(cmd) + grpcBindAddr, grpcGracePeriod, grpcCert, grpcKey, grpcClientCA := regGRPCFlags(cmd) promURL := cmd.Flag("prometheus.url", "URL at which to reach Prometheus's API. For better performance use local network."). Default("http://localhost:9090").URL() @@ -79,9 +78,10 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { reg, tracer, *grpcBindAddr, - *cert, - *key, - *clientCA, + time.Duration(*grpcGracePeriod), + *grpcCert, + *grpcKey, + *grpcClientCA, *httpBindAddr, time.Duration(*httpGracePeriod), *promURL, @@ -102,9 +102,10 @@ func runSidecar( reg *prometheus.Registry, tracer opentracing.Tracer, grpcBindAddr string, - cert string, - key string, - clientCA string, + grpcGracePeriod time.Duration, + grpcCert string, + grpcKey string, + grpcClientCA string, httpBindAddr string, httpGracePeriod time.Duration, promURL *url.URL, @@ -140,9 +141,9 @@ func runSidecar( // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) - srv := server.NewHTTP(logger, reg, comp, statusProber, - server.WithListen(httpBindAddr), - server.WithGracePeriod(httpGracePeriod), + srv := httpserver.New(logger, reg, comp, statusProber, + httpserver.WithListen(httpBindAddr), + httpserver.WithGracePeriod(httpGracePeriod), ) g.Add(srv.ListenAndServe, srv.Shutdown) @@ -229,29 +230,28 @@ func runSidecar( } { - l, err := net.Listen("tcp", grpcBindAddr) - if err != nil { - return errors.Wrap(err, "listen API address") - } - logger := log.With(logger, "component", component.Sidecar.String()) - promStore, err := store.NewPrometheusStore( logger, nil, promURL, component.Sidecar, m.Labels, m.Timestamps) if err != nil { return errors.Wrap(err, "create Prometheus store") } - opts, err := defaultGRPCTLSServerOpts(logger, cert, key, clientCA) + tlsCfg, err := tls.NewServerConfig(log.With(logger, "protocol", "gRPC"), grpcCert, grpcKey, grpcClientCA) if err != nil { return errors.Wrap(err, "setup gRPC server") } - s := newStoreGRPCServer(logger, reg, tracer, promStore, opts) + s := grpcserver.New(logger, reg, tracer, comp, promStore, + grpcserver.WithListen(grpcBindAddr), + grpcserver.WithGracePeriod(grpcGracePeriod), + grpcserver.WithTLSConfig(tlsCfg), + ) g.Add(func() error { - level.Info(logger).Log("msg", "Listening for StoreAPI gRPC", "address", grpcBindAddr) - return errors.Wrap(s.Serve(l), "serve gRPC") - }, func(error) { - s.Stop() + statusProber.SetReady() + return s.ListenAndServe() + }, func(err error) { + statusProber.SetNotReady(err) + s.Shutdown(err) }) } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 54e8a63124..13833a3a12 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -2,12 +2,8 @@ package main import ( "context" - "net" "time" - "github.com/thanos-io/thanos/pkg/extflag" - "github.com/thanos-io/thanos/pkg/server" - "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -16,12 +12,16 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/relabel" "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/extflag" "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/runutil" + grpcserver "github.com/thanos-io/thanos/pkg/server/grpc" + httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/store" storecache "github.com/thanos-io/thanos/pkg/store/cache" + "github.com/thanos-io/thanos/pkg/tls" "gopkg.in/alecthomas/kingpin.v2" yaml "gopkg.in/yaml.v2" ) @@ -30,9 +30,8 @@ import ( func registerStore(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(component.Store.String(), "store node giving access to blocks in a bucket provider. Now supported GCS, S3, Azure, Swift and Tencent COS.") - httpBindAddr := regHTTPAddrFlag(cmd) - httpGracePeriod := regHTTPGracePeriodFlag(cmd) - grpcBindAddr, cert, key, clientCA := regGRPCFlags(cmd) + httpBindAddr, httpGracePeriod := regHTTPFlags(cmd) + grpcBindAddr, grpcGracePeriod, grpcCert, grpcKey, grpcClientCA := regGRPCFlags(cmd) dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() @@ -81,9 +80,10 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { objStoreConfig, *dataDir, *grpcBindAddr, - *cert, - *key, - *clientCA, + time.Duration(*grpcGracePeriod), + *grpcCert, + *grpcKey, + *grpcClientCA, *httpBindAddr, time.Duration(*httpGracePeriod), uint64(*indexCacheSize), @@ -113,9 +113,10 @@ func runStore( objStoreConfig *extflag.PathOrContent, dataDir string, grpcBindAddr string, - cert string, - key string, - clientCA string, + grpcGracePeriod time.Duration, + grpcCert string, + grpcKey string, + grpcClientCA string, httpBindAddr string, httpGracePeriod time.Duration, indexCacheSizeBytes uint64, @@ -132,9 +133,9 @@ func runStore( ) error { // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) - srv := server.NewHTTP(logger, reg, component, statusProber, - server.WithListen(httpBindAddr), - server.WithGracePeriod(httpGracePeriod), + srv := httpserver.New(logger, reg, component, statusProber, + httpserver.WithListen(httpBindAddr), + httpserver.WithGracePeriod(httpGracePeriod), ) g.Add(srv.ListenAndServe, srv.Shutdown) @@ -225,26 +226,28 @@ func runStore( cancel() }) } + // Start query (proxy) gRPC StoreAPI. + { + tlsCfg, err := tls.NewServerConfig(log.With(logger, "protocol", "gRPC"), grpcCert, grpcKey, grpcClientCA) + if err != nil { + return errors.Wrap(err, "setup gRPC server") + } - l, err := net.Listen("tcp", grpcBindAddr) - if err != nil { - return errors.Wrap(err, "listen API address") - } + s := grpcserver.New(logger, reg, tracer, component, bs, + grpcserver.WithListen(grpcBindAddr), + grpcserver.WithGracePeriod(grpcGracePeriod), + grpcserver.WithTLSConfig(tlsCfg), + ) - opts, err := defaultGRPCTLSServerOpts(logger, cert, key, clientCA) - if err != nil { - return errors.Wrap(err, "grpc server options") + g.Add(func() error { + <-bucketStoreReady + statusProber.SetReady() + return s.ListenAndServe() + }, func(err error) { + statusProber.SetNotReady(err) + s.Shutdown(err) + }) } - s := newStoreGRPCServer(logger, reg, tracer, bs, opts) - - g.Add(func() error { - <-bucketStoreReady - level.Info(logger).Log("msg", "listening for StoreAPI gRPC", "address", grpcBindAddr) - statusProber.SetReady() - return errors.Wrap(s.Serve(l), "serve gRPC") - }, func(error) { - s.Stop() - }) level.Info(logger).Log("msg", "starting store node") return nil diff --git a/docs/components/bucket.md b/docs/components/bucket.md index df6dcd284d..234a5234ac 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -122,7 +122,9 @@ Flags: object store configuration. See format details: https://thanos.io/storage.md/#configuration --listen="0.0.0.0:8080" HTTP host:port to listen on - --http-grace-period=5s Time to wait after an interrupt received for HTTP + --http-address="0.0.0.0:10902" + Listen host:port for HTTP endpoints. + --http-grace-period=2m Time to wait after an interrupt received for HTTP Server. --refresh=30m Refresh interval to download metadata from remote storage diff --git a/docs/components/compact.md b/docs/components/compact.md index df1056f571..4a710a6773 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -84,7 +84,7 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. - --http-grace-period=5s Time to wait after an interrupt received for HTTP + --http-grace-period=2m Time to wait after an interrupt received for HTTP Server. --data-dir="./data" Data directory in which to cache blocks and process compactions. diff --git a/docs/components/query.md b/docs/components/query.md index 3ebf692471..2f0a29771f 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -259,12 +259,14 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. - --http-grace-period=5s Time to wait after an interrupt received for + --http-grace-period=2m Time to wait after an interrupt received for HTTP Server. --grpc-address="0.0.0.0:10901" Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable from other components. + --grpc-grace-period=2m Time to wait after an interrupt received for + GRPC Server. --grpc-server-tls-cert="" TLS Certificate for gRPC server, leave blank to disable TLS --grpc-server-tls-key="" TLS Key for the gRPC server, leave blank to diff --git a/docs/components/rule.md b/docs/components/rule.md index a697dc02fb..55bac66749 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -168,12 +168,14 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. - --http-grace-period=5s Time to wait after an interrupt received for + --http-grace-period=2m Time to wait after an interrupt received for HTTP Server. --grpc-address="0.0.0.0:10901" Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable from other components. + --grpc-grace-period=2m Time to wait after an interrupt received for + GRPC Server. --grpc-server-tls-cert="" TLS Certificate for gRPC server, leave blank to disable TLS --grpc-server-tls-key="" TLS Key for the gRPC server, leave blank to diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 5546ad20f2..4dfc4282b2 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -101,12 +101,14 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. - --http-grace-period=5s Time to wait after an interrupt received for + --http-grace-period=2m Time to wait after an interrupt received for HTTP Server. --grpc-address="0.0.0.0:10901" Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable from other components. + --grpc-grace-period=2m Time to wait after an interrupt received for + GRPC Server. --grpc-server-tls-cert="" TLS Certificate for gRPC server, leave blank to disable TLS --grpc-server-tls-key="" TLS Key for the gRPC server, leave blank to diff --git a/docs/components/store.md b/docs/components/store.md index 6f11cebc9d..3b2a7f5737 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -51,12 +51,14 @@ Flags: https://thanos.io/tracing.md/#configuration --http-address="0.0.0.0:10902" Listen host:port for HTTP endpoints. - --http-grace-period=5s Time to wait after an interrupt received for + --http-grace-period=2m Time to wait after an interrupt received for HTTP Server. --grpc-address="0.0.0.0:10901" Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable from other components. + --grpc-grace-period=2m Time to wait after an interrupt received for + GRPC Server. --grpc-server-tls-cert="" TLS Certificate for gRPC server, leave blank to disable TLS --grpc-server-tls-key="" TLS Key for the gRPC server, leave blank to diff --git a/go.mod b/go.mod index 62b62543c4..454676ed2b 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.2.1 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 - github.com/prometheus/common v0.6.0 + github.com/prometheus/common v0.7.0 github.com/prometheus/prometheus v1.8.2-0.20190913102521-8ab628b35467 // v1.8.2 is misleading as Prometheus does not have v2 module. github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible @@ -49,7 +49,6 @@ require ( golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20190423024810-112230192c58 - golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect golang.org/x/text v0.3.2 google.golang.org/api v0.11.0 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 diff --git a/go.sum b/go.sum index f57134141b..2ec5b5a3cd 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v0.0.0-20181017004759-096ff4a8a059/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= @@ -402,6 +403,7 @@ github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -410,6 +412,7 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/prometheus v1.8.2-0.20190913102521-8ab628b35467 h1:B9IMa7s163/ZDSduepHHfOZZHSKdSbgo/bFY5c+FMAs= diff --git a/pkg/server/grpc/grpc.go b/pkg/server/grpc/grpc.go new file mode 100644 index 0000000000..29774485eb --- /dev/null +++ b/pkg/server/grpc/grpc.go @@ -0,0 +1,132 @@ +package grpc + +import ( + "context" + "math" + "net" + "runtime/debug" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/tracing" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" +) + +// A Server defines parameters to serve RPC requests, a wrapper around grpc.Server. +type Server struct { + logger log.Logger + comp component.Component + + srv *grpc.Server + listener net.Listener + + opts options +} + +// New creates a new Server. +func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, comp component.Component, storeSrv storepb.StoreServer, opts ...Option) *Server { + options := options{} + for _, o := range opts { + o.apply(&options) + } + + met := grpc_prometheus.NewServerMetrics() + met.EnableHandlingTimeHistogram( + grpc_prometheus.WithHistogramBuckets([]float64{ + 0.001, 0.01, 0.05, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, + }), + ) + panicsTotal := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "thanos_grpc_req_panics_recovered_total", + Help: "Total number of gRPC requests recovered from internal panic.", + }) + reg.MustRegister(met, panicsTotal) + + grpcPanicRecoveryHandler := func(p interface{}) (err error) { + panicsTotal.Inc() + level.Error(logger).Log("msg", "recovered from panic", "panic", p, "stack", debug.Stack()) + return status.Errorf(codes.Internal, "%s", p) + } + + grpcOpts := []grpc.ServerOption{} + grpcOpts = append(grpcOpts, + grpc.MaxSendMsgSize(math.MaxInt32), + grpc_middleware.WithUnaryServerChain( + met.UnaryServerInterceptor(), + tracing.UnaryServerInterceptor(tracer), + grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), + ), + grpc_middleware.WithStreamServerChain( + met.StreamServerInterceptor(), + tracing.StreamServerInterceptor(tracer), + grpc_recovery.StreamServerInterceptor(grpc_recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), + ), + ) + + if options.tlsConfig != nil { + grpcOpts = append(grpcOpts, grpc.Creds(credentials.NewTLS(options.tlsConfig))) + } + s := grpc.NewServer(grpcOpts...) + + storepb.RegisterStoreServer(s, storeSrv) + met.InitializeMetrics(s) + + return &Server{ + logger: log.With(logger, "service", "gRPC/server", "component", comp.String()), + comp: comp, + srv: s, + opts: options, + } +} + +// ListenAndServe listens on the TCP network address and handles requests on incoming connections. +func (s *Server) ListenAndServe() error { + l, err := net.Listen("tcp", s.opts.listen) + if err != nil { + return errors.Wrapf(err, "listen gRPC on address %s", s.opts.listen) + } + s.listener = l + + level.Info(s.logger).Log("msg", "listening for StoreAPI gRPC", "address", s.opts.listen) + return errors.Wrap(s.srv.Serve(s.listener), "serve gRPC") +} + +// Shutdown gracefully shuts down the server by waiting, +// for specified amount of time (by gracePeriod) for connections to return to idle and then shut down. +func (s *Server) Shutdown(err error) { + defer level.Info(s.logger).Log("msg", "internal server shutdown", "err", err) + + if s.opts.gracePeriod == 0 { + s.srv.Stop() + return + } + + ctx, cancel := context.WithTimeout(context.Background(), s.opts.gracePeriod) + defer cancel() + + stopped := make(chan struct{}) + go func() { + level.Info(s.logger).Log("msg", "gracefully stopping internal server") + s.srv.GracefulStop() // Also closes s.listener. + close(stopped) + }() + + select { + case <-ctx.Done(): + level.Info(s.logger).Log("msg", "grace period exceeded enforcing shutdown") + s.srv.Stop() + case <-stopped: + cancel() + } +} diff --git a/pkg/server/grpc/option.go b/pkg/server/grpc/option.go new file mode 100644 index 0000000000..6c804e4f79 --- /dev/null +++ b/pkg/server/grpc/option.go @@ -0,0 +1,47 @@ +package grpc + +import ( + "crypto/tls" + "time" +) + +type options struct { + gracePeriod time.Duration + listen string + + tlsConfig *tls.Config +} + +// Option overrides behavior of Server. +type Option interface { + apply(*options) +} + +type optionFunc func(*options) + +func (f optionFunc) apply(o *options) { + f(o) +} + +// WithGracePeriod sets shutdown grace period for gRPC server. +// Server waits connections to drain for specified amount of time. +func WithGracePeriod(t time.Duration) Option { + return optionFunc(func(o *options) { + o.gracePeriod = t + }) +} + +// WithListen sets address to listen for gRPC server. +// Server accepts incoming connections on given address. +func WithListen(s string) Option { + return optionFunc(func(o *options) { + o.listen = s + }) +} + +// WithTLSConfig sets TLS configuration for gRPC server. +func WithTLSConfig(cfg *tls.Config) Option { + return optionFunc(func(o *options) { + o.tlsConfig = cfg + }) +} diff --git a/pkg/server/http.go b/pkg/server/http/http.go similarity index 64% rename from pkg/server/http.go rename to pkg/server/http/http.go index d7b17a5a77..290d97c59a 100644 --- a/pkg/server/http.go +++ b/pkg/server/http/http.go @@ -1,21 +1,20 @@ -package server +package http import ( "context" "net/http" "net/http/pprof" - "time" - - "github.com/thanos-io/thanos/pkg/component" - "github.com/thanos-io/thanos/pkg/prober" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/prober" ) +// A Server defines parameters for serve HTTP requests, a wrapper around http.Server. type Server struct { logger log.Logger comp component.Component @@ -27,12 +26,9 @@ type Server struct { opts options } -func NewHTTP(logger log.Logger, reg *prometheus.Registry, comp component.Component, prober *prober.Prober, opts ...Option) Server { - options := options{ - gracePeriod: 5 * time.Second, - listen: "0.0.0.0:10902", - } - +// New creates a new Server. +func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, prober *prober.Prober, opts ...Option) *Server { + options := options{} for _, o := range opts { o.apply(&options) } @@ -42,8 +38,8 @@ func NewHTTP(logger log.Logger, reg *prometheus.Registry, comp component.Compone registerProfiler(mux) prober.RegisterInMux(mux) - return Server{ - logger: log.With(logger, "service", "http/server"), + return &Server{ + logger: log.With(logger, "service", "http/server", "component", comp.String()), comp: comp, prober: prober, mux: mux, @@ -52,12 +48,15 @@ func NewHTTP(logger log.Logger, reg *prometheus.Registry, comp component.Compone } } +// ListenAndServe listens on the TCP network address and handles requests on incoming connections. func (s *Server) ListenAndServe() error { s.prober.SetHealthy() - level.Info(s.logger).Log("msg", "listening for requests and metrics", "component", s.comp.String(), "address", s.opts.listen) - return errors.Wrapf(s.srv.ListenAndServe(), "serve %s and metrics", s.comp.String()) + level.Info(s.logger).Log("msg", "listening for requests and metrics", "address", s.opts.listen) + return errors.Wrap(s.srv.ListenAndServe(), "serve HTTP and metrics") } +// Shutdown gracefully shuts down the server by waiting, +// for specified amount of time (by gracePeriod) for connections to return to idle and then shut down. func (s *Server) Shutdown(err error) { s.prober.SetNotReady(err) defer s.prober.SetNotHealthy(err) @@ -67,16 +66,22 @@ func (s *Server) Shutdown(err error) { return } + defer level.Info(s.logger).Log("msg", "internal server shutdown", "err", err) + + if s.opts.gracePeriod == 0 { + s.srv.Close() + return + } + ctx, cancel := context.WithTimeout(context.Background(), s.opts.gracePeriod) defer cancel() - level.Info(s.logger).Log("msg", "server shut down internal server") - if err := s.srv.Shutdown(ctx); err != nil { - level.Error(s.logger).Log("msg", "server shut down failed", "err", err, "component", s.comp.String()) + level.Error(s.logger).Log("msg", "internal server shut down failed", "err", err) } } +// Handle registers the handler for the given pattern. func (s *Server) Handle(pattern string, handler http.Handler) { s.mux.Handle(pattern, handler) } diff --git a/pkg/server/option.go b/pkg/server/http/option.go similarity index 64% rename from pkg/server/option.go rename to pkg/server/http/option.go index 702b6dbecc..ab7ff59e0c 100644 --- a/pkg/server/option.go +++ b/pkg/server/http/option.go @@ -1,4 +1,4 @@ -package server +package http import ( "time" @@ -20,12 +20,16 @@ func (f optionFunc) apply(o *options) { f(o) } +// WithGracePeriod sets shutdown grace period for HTTP server. +// Server waits connections to drain for specified amount of time. func WithGracePeriod(t time.Duration) Option { return optionFunc(func(o *options) { o.gracePeriod = t }) } +// WithListen sets address to listen for HTTP server. +// Server accepts incoming TCP connections on given address. func WithListen(s string) Option { return optionFunc(func(o *options) { o.listen = s diff --git a/pkg/tls/options.go b/pkg/tls/options.go new file mode 100644 index 0000000000..58b6c56d36 --- /dev/null +++ b/pkg/tls/options.go @@ -0,0 +1,104 @@ +package tls + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" +) + +// NewServerConfig provides new server TLS configuration. +func NewServerConfig(logger log.Logger, cert, key, clientCA string) (*tls.Config, error) { + if key == "" && cert == "" { + if clientCA != "" { + return nil, errors.New("when a client CA is used a server key and certificate must also be provided") + } + + level.Info(logger).Log("msg", "disabled TLS, key and cert must be set to enable") + return nil, nil + } + + level.Info(logger).Log("msg", "enabling server side TLS") + + if key == "" || cert == "" { + return nil, errors.New("both server key and certificate must be provided") + } + + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + } + + tlsCert, err := tls.LoadX509KeyPair(cert, key) + if err != nil { + return nil, errors.Wrap(err, "server credentials") + } + + tlsCfg.Certificates = []tls.Certificate{tlsCert} + + if clientCA != "" { + caPEM, err := ioutil.ReadFile(clientCA) + if err != nil { + return nil, errors.Wrap(err, "reading client CA") + } + + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caPEM) { + return nil, errors.Wrap(err, "building client CA") + } + tlsCfg.ClientCAs = certPool + tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert + + level.Info(logger).Log("msg", "server TLS client verification enabled") + } + + return tlsCfg, nil +} + +// NewClientConfig provides new client TLS configuration. +func NewClientConfig(logger log.Logger, cert, key, caCert, serverName string) (*tls.Config, error) { + var certPool *x509.CertPool + if caCert != "" { + caPEM, err := ioutil.ReadFile(caCert) + if err != nil { + return nil, errors.Wrap(err, "reading client CA") + } + + certPool = x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caPEM) { + return nil, errors.Wrap(err, "building client CA") + } + level.Info(logger).Log("msg", "TLS client using provided certificate pool") + } else { + var err error + certPool, err = x509.SystemCertPool() + if err != nil { + return nil, errors.Wrap(err, "reading system certificate pool") + } + level.Info(logger).Log("msg", "TLS client using system certificate pool") + } + + tlsCfg := &tls.Config{ + RootCAs: certPool, + } + + if serverName != "" { + tlsCfg.ServerName = serverName + } + + if (key != "") != (cert != "") { + return nil, errors.New("both client key and certificate must be provided") + } + + if cert != "" { + cert, err := tls.LoadX509KeyPair(cert, key) + if err != nil { + return nil, errors.Wrap(err, "client credentials") + } + tlsCfg.Certificates = []tls.Certificate{cert} + level.Info(logger).Log("msg", "TLS client authentication enabled") + } + return tlsCfg, nil +} diff --git a/pkg/ui/bucket.go b/pkg/ui/bucket.go index 84dfdbfdb5..f1fdd8ee47 100644 --- a/pkg/ui/bucket.go +++ b/pkg/ui/bucket.go @@ -7,7 +7,7 @@ import ( "github.com/go-kit/kit/log" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" - "github.com/thanos-io/thanos/pkg/server" + httpserver "github.com/thanos-io/thanos/pkg/server/http" ) // Bucket is a web UI representing state of buckets as a timeline. @@ -30,7 +30,7 @@ func NewBucketUI(logger log.Logger, label string) *Bucket { } // Register registers http routes for bucket UI. -func (b *Bucket) Register(s server.Server, ins extpromhttp.InstrumentationMiddleware) { +func (b *Bucket) Register(s *httpserver.Server, ins extpromhttp.InstrumentationMiddleware) { instrf := func(name string, next func(w http.ResponseWriter, r *http.Request)) http.HandlerFunc { return ins.NewHandler(name, http.HandlerFunc(next)) } diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh index faff7bcc76..ccdee968b6 100755 --- a/scripts/quickstart.sh +++ b/scripts/quickstart.sh @@ -23,8 +23,7 @@ fi # Start local object storage, if desired. # NOTE: If you would like to use an actual S3-compatible API with this setup # set the S3_* environment variables set in the Minio example. -if [ -n "${MINIO_ENABLED}" ] -then +if [ -n "${MINIO_ENABLED}" ]; then if [ ! $(command -v $MINIO_EXECUTABLE) ]; then echo "Cannot find or execute Minio binary $MINIO_EXECUTABLE, you can override it by setting the MINIO_EXECUTABLE env variable" exit 1 @@ -49,14 +48,14 @@ then mkdir -p data/minio ${MINIO_EXECUTABLE} server ./data/minio \ - --address ${MINIO_ENDPOINT} & + --address ${MINIO_ENDPOINT} & sleep 3 # create the bucket ${MC_EXECUTABLE} config host add tmp http://${MINIO_ENDPOINT} ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} ${MC_EXECUTABLE} mb tmp/${MINIO_BUCKET} ${MC_EXECUTABLE} config host rm tmp - cat < data/bucket.yml + cat <data/bucket.yml type: S3 config: bucket: $S3_BUCKET @@ -71,12 +70,11 @@ fi STORES="" # Start three Prometheus servers monitoring themselves. -for i in `seq 0 2` -do +for i in $(seq 0 2); do rm -rf data/prom${i} mkdir -p data/prom${i}/ - cat > data/prom${i}/prometheus.yml <<- EOF + cat >data/prom${i}/prometheus.yml <<-EOF global: external_labels: prometheus: prom-${i} @@ -110,13 +108,13 @@ scrape_configs: EOF ${PROMETHEUS_EXECUTABLE} \ - --config.file data/prom${i}/prometheus.yml \ - --storage.tsdb.path data/prom${i} \ - --log.level warn \ + --config.file data/prom${i}/prometheus.yml \ + --storage.tsdb.path data/prom${i} \ + --log.level warn \ --web.enable-lifecycle \ --storage.tsdb.min-block-duration=2h \ --storage.tsdb.max-block-duration=2h \ - --web.listen-address 0.0.0.0:909${i} & + --web.listen-address 0.0.0.0:909${i} & sleep 0.25 done @@ -124,20 +122,21 @@ done sleep 0.5 OBJSTORECFG="" -if [ -n "${MINIO_ENABLED}" ] -then -OBJSTORECFG="--objstore.config-file data/bucket.yml" +if [ -n "${MINIO_ENABLED}" ]; then + OBJSTORECFG="--objstore.config-file data/bucket.yml" fi # Start one sidecar for each Prometheus server. -for i in `seq 0 2` -do +for i in $(seq 0 2); do ${THANOS_EXECUTABLE} sidecar \ - --debug.name sidecar-${i} \ - --grpc-address 0.0.0.0:109${i}1 \ - --http-address 0.0.0.0:109${i}2 \ - --prometheus.url http://localhost:909${i} \ - --tsdb.path data/prom${i} \ + --debug.name sidecar-${i} \ + --log.level debug \ + --grpc-address 0.0.0.0:109${i}1 \ + --grpc-grace-period 1s \ + --http-address 0.0.0.0:109${i}2 \ + --http-grace-period 1s \ + --prometheus.url http://localhost:909${i} \ + --tsdb.path data/prom${i} \ ${OBJSTORECFG} & STORES="${STORES} --store 127.0.0.1:109${i}1" @@ -147,14 +146,15 @@ done sleep 0.5 -if [ -n "${GCS_BUCKET}" -o -n "${S3_ENDPOINT}" ] -then +if [ -n "${GCS_BUCKET}" -o -n "${S3_ENDPOINT}" ]; then ${THANOS_EXECUTABLE} store \ - --debug.name store \ - --log.level debug \ - --grpc-address 0.0.0.0:10905 \ - --http-address 0.0.0.0:10906 \ - --data-dir data/store \ + --debug.name store \ + --log.level debug \ + --grpc-address 0.0.0.0:10905 \ + --grpc-grace-period 1s \ + --http-address 0.0.0.0:10906 \ + --http-grace-period 1s \ + --data-dir data/store \ ${OBJSTORECFG} & STORES="${STORES} --store 127.0.0.1:10905" @@ -162,20 +162,22 @@ fi sleep 0.5 -if [ -n "${REMOTE_WRITE_ENABLED}" ] -then +if [ -n "${REMOTE_WRITE_ENABLED}" ]; then ${THANOS_EXECUTABLE} receive \ - --debug.name receive \ - --log.level debug \ - --tsdb.path "./data/remote-write-receive-data" \ - --grpc-address 0.0.0.0:10907 \ - --http-address 0.0.0.0:10909 \ - --label "receive=\"true\"" \ + --debug.name receive \ + --log.level debug \ + --log.level debug \ + --tsdb.path "./data/remote-write-receive-data" \ + --grpc-address 0.0.0.0:10907 \ + --grpc-grace-period 1s \ + --http-address 0.0.0.0:10909 \ + --http-grace-period 1s \ + --label "receive=\"true\"" \ ${OBJSTORECFG} \ - --remote-write.address 0.0.0.0:10908 & + --remote-write.address 0.0.0.0:10908 & mkdir -p "data/local-prometheus-data/" - cat < data/local-prometheus-data/prometheus.yml + cat <data/local-prometheus-data/prometheus.yml # When the Thanos remote-write-receive component is started, # this is an example configuration of a Prometheus server that # would scrape a local node-exporter and replicate its data to @@ -198,15 +200,16 @@ fi sleep 0.5 # Start two query nodes. -for i in `seq 0 1` -do +for i in $(seq 0 1); do ${THANOS_EXECUTABLE} query \ - --debug.name query-${i} \ - --grpc-address 0.0.0.0:109${i}3 \ - --http-address 0.0.0.0:109${i}4 \ - --query.replica-label prometheus \ + --debug.name query-${i} \ + --log.level debug \ + --grpc-address 0.0.0.0:109${i}3 \ + --grpc-grace-period 1s \ + --http-address 0.0.0.0:109${i}4 \ + --http-grace-period 1s \ + --query.replica-label prometheus \ ${STORES} & done wait - diff --git a/test/e2e/spinup_test.go b/test/e2e/spinup_test.go index 65c636771d..9c6b3b09ad 100644 --- a/test/e2e/spinup_test.go +++ b/test/e2e/spinup_test.go @@ -156,6 +156,7 @@ func sidecar(http, grpc address, prom *prometheusScheduler) *serverScheduler { return newCmdExec(exec.Command("thanos", "sidecar", "--debug.name", fmt.Sprintf("sidecar-%s", http.Port), "--grpc-address", grpc.HostPort(), + "--grpc-grace-period", "0s", "--http-address", http.HostPort(), "--prometheus.url", prom.HTTP.URL(), "--tsdb.path", promDir, @@ -190,6 +191,7 @@ func receiver(http, grpc, metric address, replicationFactor int, hashring ...rec return newCmdExec(exec.Command("thanos", "receive", "--debug.name", fmt.Sprintf("receive-%s", http.Port), "--grpc-address", grpc.HostPort(), + "--grpc-grace-period", "0s", "--http-address", metric.HostPort(), "--remote-write.address", http.HostPort(), "--label", fmt.Sprintf(`receive="%s"`, http.Port), @@ -213,6 +215,7 @@ func querier(http, grpc address, storeAddresses []address, fileSDStoreAddresses "query", "--debug.name", fmt.Sprintf("querier-%s", http.Port), "--grpc-address", grpc.HostPort(), + "--grpc-grace-period", "0s", "--http-address", http.HostPort(), "--log.level", "debug", "--query.replica-label", replicaLabel, @@ -259,6 +262,7 @@ func storeGateway(http, grpc address, bucketConfig []byte, relabelConfig []byte) "--debug.name", fmt.Sprintf("store-gw-%s", http.Port), "--data-dir", dbDir, "--grpc-address", grpc.HostPort(), + "--grpc-grace-period", "0s", "--http-address", http.HostPort(), "--log.level", "debug", "--objstore.config", string(bucketConfig), @@ -332,6 +336,7 @@ func ruleWithDir(http, grpc address, dir string, rules []string, am address, que "--eval-interval", "1s", "--alertmanagers.url", am.URL(), "--grpc-address", grpc.HostPort(), + "--grpc-grace-period", "0s", "--http-address", http.HostPort(), "--log.level", "debug", "--query.sd-dns-interval", "5s", From d22d1ae8bfdd6bc99dcf15e611f050d24d12a549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Mon, 4 Nov 2019 12:24:20 +0100 Subject: [PATCH 035/257] cmd/thanos/bucket: remove unused flag (#1712) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While standardizing some flags, commit 9d4d0bf408ed86eaab371c8f0c52d7bb5fe6e164 accidentally added a `--http-address` flag to the bucket web component that is never used. However, rather than remove this new flag, this commit removes the `--listen` flag that the component defines so that the component is in line with the flags defined by other components. Signed-off-by: Lucas Servén Marín --- CHANGELOG.md | 1 + cmd/thanos/bucket.go | 5 ++-- docs/components/bucket.md | 53 +++++++++++++++++++-------------------- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68be8ef781..b8f4785457 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. - [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss) for further information. - [#1680](https://github.com/thanos-io/thanos/pull/1680) Add a new `--http-grace-period` CLI option to components which serve HTTP to set how long to wait until HTTP Server shuts down. +- [#1712](https://github.com/thanos-io/thanos/pull/1712) Rename flag on bucket web component from `--listen` to `--http-address` to match other components. ### Fixed diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 244d8942a1..9575a8c406 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -309,8 +309,7 @@ func registerBucketInspect(m map[string]setupFunc, root *kingpin.CmdClause, name // registerBucketWeb exposes a web interface for the state of remote store like `pprof web`. func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) { cmd := root.Command("web", "Web interface for remote storage bucket") - bind := cmd.Flag("listen", "HTTP host:port to listen on").Default("0.0.0.0:8080").String() - _, httpGracePeriod := regHTTPFlags(cmd) + httpBindAddr, httpGracePeriod := regHTTPFlags(cmd) interval := cmd.Flag("refresh", "Refresh interval to download metadata from remote storage").Default("30m").Duration() timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() label := cmd.Flag("label", "Prometheus label to use as timeline title").String() @@ -321,7 +320,7 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str statusProber := prober.NewProber(component.Bucket, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. srv := httpserver.New(logger, reg, component.Bucket, statusProber, - httpserver.WithListen(*bind), + httpserver.WithListen(*httpBindAddr), httpserver.WithGracePeriod(time.Duration(*httpGracePeriod)), ) diff --git a/docs/components/bucket.md b/docs/components/bucket.md index 234a5234ac..edcff62cfa 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -98,38 +98,37 @@ usage: thanos bucket web [] Web interface for remote storage bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + -h, --help Show context-sensitive help (also try --help-long + and --help-man). + --version Show application version. + --log.level=info Log filtering level. + --log.format=logfmt Log format to use. --tracing.config-file= - Path to YAML file with tracing configuration. See - format details: - https://thanos.io/tracing.md/#configuration + Path to YAML file with tracing configuration. See + format details: + https://thanos.io/tracing.md/#configuration --tracing.config= - Alternative to 'tracing.config-file' flag (lower - priority). Content of YAML file with tracing - configuration. See format details: - https://thanos.io/tracing.md/#configuration + Alternative to 'tracing.config-file' flag (lower + priority). Content of YAML file with tracing + configuration. See format details: + https://thanos.io/tracing.md/#configuration --objstore.config-file= - Path to YAML file that contains object store - configuration. See format details: - https://thanos.io/storage.md/#configuration + Path to YAML file that contains object store + configuration. See format details: + https://thanos.io/storage.md/#configuration --objstore.config= - Alternative to 'objstore.config-file' flag (lower - priority). Content of YAML file that contains - object store configuration. See format details: - https://thanos.io/storage.md/#configuration - --listen="0.0.0.0:8080" HTTP host:port to listen on + Alternative to 'objstore.config-file' flag (lower + priority). Content of YAML file that contains + object store configuration. See format details: + https://thanos.io/storage.md/#configuration --http-address="0.0.0.0:10902" - Listen host:port for HTTP endpoints. - --http-grace-period=2m Time to wait after an interrupt received for HTTP - Server. - --refresh=30m Refresh interval to download metadata from remote - storage - --timeout=5m Timeout to download metadata from remote storage - --label=LABEL Prometheus label to use as timeline title + Listen host:port for HTTP endpoints. + --http-grace-period=2m Time to wait after an interrupt received for HTTP + Server. + --refresh=30m Refresh interval to download metadata from remote + storage + --timeout=5m Timeout to download metadata from remote storage + --label=LABEL Prometheus label to use as timeline title ``` From 3bd1a65cc7cdb77a9e6bb88ade874981997896a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Mon, 4 Nov 2019 16:47:10 +0100 Subject: [PATCH 036/257] cmd/thanos/bucket: fix static fileserver (#1713) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit fixes a regression introduced in #1702, where the signature of the bucketui.Register method was changed. The bug was caused because one of the registered paths relied on the HTTP parameter variable support from github.com/julienschmidt/httprouter, which the standard library does not support. The fix is instead to implement the change recommended in https://github.com/thanos-io/thanos/pull/1702#discussion_r341269556. Signed-off-by: Lucas Servén Marín --- cmd/thanos/bucket.go | 5 ++++- pkg/ui/bucket.go | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 9575a8c406..3a7646501c 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -18,6 +18,7 @@ import ( opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/route" "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" @@ -324,8 +325,10 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str httpserver.WithGracePeriod(time.Duration(*httpGracePeriod)), ) + router := route.New() bucketUI := ui.NewBucketUI(logger, *label) - bucketUI.Register(srv, extpromhttp.NewInstrumentationMiddleware(reg)) + bucketUI.Register(router, extpromhttp.NewInstrumentationMiddleware(reg)) + srv.Handle("/", router) if *interval < 5*time.Minute { level.Warn(logger).Log("msg", "Refreshing more often than 5m could lead to large data transfers") diff --git a/pkg/ui/bucket.go b/pkg/ui/bucket.go index f1fdd8ee47..8e023ba61b 100644 --- a/pkg/ui/bucket.go +++ b/pkg/ui/bucket.go @@ -6,8 +6,8 @@ import ( "time" "github.com/go-kit/kit/log" + "github.com/prometheus/common/route" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" - httpserver "github.com/thanos-io/thanos/pkg/server/http" ) // Bucket is a web UI representing state of buckets as a timeline. @@ -30,13 +30,13 @@ func NewBucketUI(logger log.Logger, label string) *Bucket { } // Register registers http routes for bucket UI. -func (b *Bucket) Register(s *httpserver.Server, ins extpromhttp.InstrumentationMiddleware) { +func (b *Bucket) Register(r *route.Router, ins extpromhttp.InstrumentationMiddleware) { instrf := func(name string, next func(w http.ResponseWriter, r *http.Request)) http.HandlerFunc { return ins.NewHandler(name, http.HandlerFunc(next)) } - s.Handle("/", instrf("root", b.root)) - s.Handle("/static/*filepath", instrf("static", b.serveStaticAsset)) + r.Get("/", instrf("root", b.root)) + r.Get("/static/*filepath", instrf("static", b.serveStaticAsset)) } // Handle / of bucket UIs. From 5d936e23acc606774754baf4448be4b12b7d976f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Tue, 5 Nov 2019 14:21:15 +0100 Subject: [PATCH 037/257] pkg/receive: eliminate duplicate tracing (#1717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The receive component currently has duplicate spans for requests handled by its HTTP server. This is due to the fact that the server is instrumented once using the global singleton tracer and then a second time with the tracer configured via the CLI. This commit eliminates the duplicate instrumentation via the global singleton in favor of the explicit specified tracer, which is consistent with the practice of other Thanos components. Signed-off-by: Lucas Servén Marín --- go.mod | 1 - pkg/receive/handler.go | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 454676ed2b..9ab5c65838 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,6 @@ require ( github.com/oklog/run v1.0.0 github.com/oklog/ulid v1.3.1 github.com/olekukonko/tablewriter v0.0.1 - github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 github.com/opentracing/basictracer-go v1.0.0 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 diff --git a/pkg/receive/handler.go b/pkg/receive/handler.go index 16a988098e..7d082a9de1 100644 --- a/pkg/receive/handler.go +++ b/pkg/receive/handler.go @@ -17,7 +17,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/golang/snappy" conntrack "github.com/mwitkow/go-conntrack" - "github.com/opentracing-contrib/go-stdlib/nethttp" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -182,16 +181,10 @@ func (h *Handler) Run() error { conntrack.TrackWithName("http"), conntrack.TrackWithTracing()) - operationName := nethttp.OperationNameFunc(func(r *http.Request) string { - return fmt.Sprintf("%s %s", r.Method, r.URL.Path) - }) - mux := http.NewServeMux() - mux.Handle("/", h.router) - errlog := stdlog.New(log.NewStdlibAdapter(level.Error(h.logger)), "", 0) httpSrv := &http.Server{ - Handler: nethttp.Middleware(opentracing.GlobalTracer(), mux, operationName), + Handler: h.router, ErrorLog: errlog, TLSConfig: h.options.TLSConfig, } From f501429d2050a0fe03dfe463d31e896c12aab142 Mon Sep 17 00:00:00 2001 From: Philip Panyukov Date: Tue, 5 Nov 2019 13:35:15 +0000 Subject: [PATCH 038/257] remove reallocs and overallocs for s.lset in bucket.go (#1718) - append to s.lset caused reallocs and overallocs - this is now fixed by properly setting capacity on s.lset - pprof shows "-316.22MB, 5.00% of 6322.82MB total" for sample data set and query I used Signed-off-by: Philip Panyukov --- pkg/store/bucket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index e1260e0ecb..3f2c409b3e 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -680,7 +680,7 @@ func blockSeries( return nil, nil, errors.Wrap(err, "read series") } s := seriesEntry{ - lset: make([]storepb.Label, 0, len(lset)), + lset: make([]storepb.Label, 0, len(lset)+len(extLset)), refs: make([]uint64, 0, len(chks)), chks: make([]storepb.AggrChunk, 0, len(chks)), } From 27a578da806557a4ccf35c4019e99143686b9661 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 5 Nov 2019 15:12:57 +0000 Subject: [PATCH 039/257] Small clean up for tracing. (#1720) Signed-off-by: Bartek Plotka --- pkg/server/grpc/grpc.go | 5 ++--- pkg/tracing/grpc.go | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/server/grpc/grpc.go b/pkg/server/grpc/grpc.go index 29774485eb..926b6068aa 100644 --- a/pkg/server/grpc/grpc.go +++ b/pkg/server/grpc/grpc.go @@ -59,8 +59,7 @@ func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer return status.Errorf(codes.Internal, "%s", p) } - grpcOpts := []grpc.ServerOption{} - grpcOpts = append(grpcOpts, + grpcOpts := []grpc.ServerOption{ grpc.MaxSendMsgSize(math.MaxInt32), grpc_middleware.WithUnaryServerChain( met.UnaryServerInterceptor(), @@ -72,7 +71,7 @@ func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer tracing.StreamServerInterceptor(tracer), grpc_recovery.StreamServerInterceptor(grpc_recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), ), - ) + } if options.tlsConfig != nil { grpcOpts = append(grpcOpts, grpc.Creds(credentials.NewTLS(options.tlsConfig))) diff --git a/pkg/tracing/grpc.go b/pkg/tracing/grpc.go index ee13810ca7..91c861ebb3 100644 --- a/pkg/tracing/grpc.go +++ b/pkg/tracing/grpc.go @@ -21,17 +21,21 @@ func StreamClientInterceptor(tracer opentracing.Tracer) grpc.StreamClientInterce // UnaryServerInterceptor returns a new unary server interceptor for OpenTracing and injects given tracer. func UnaryServerInterceptor(tracer opentracing.Tracer) grpc.UnaryServerInterceptor { + interceptor := grpc_opentracing.UnaryServerInterceptor(grpc_opentracing.WithTracer(tracer)) return func(parentCtx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - return grpc_opentracing.UnaryServerInterceptor(grpc_opentracing.WithTracer(tracer))(ContextWithTracer(parentCtx, tracer), req, info, handler) + // Add our own tracer. + return interceptor(ContextWithTracer(parentCtx, tracer), req, info, handler) } } // StreamServerInterceptor returns a new streaming server interceptor for OpenTracing and injects given tracer. func StreamServerInterceptor(tracer opentracing.Tracer) grpc.StreamServerInterceptor { + interceptor := grpc_opentracing.StreamServerInterceptor(grpc_opentracing.WithTracer(tracer)) return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + // Add our own tracer. wrappedStream := grpc_middleware.WrapServerStream(stream) wrappedStream.WrappedContext = ContextWithTracer(stream.Context(), tracer) - return grpc_opentracing.StreamServerInterceptor(grpc_opentracing.WithTracer(tracer))(srv, wrappedStream, info, handler) + return interceptor(srv, wrappedStream, info, handler) } } From 514f3088c9aa11084ab5e1a6604b8c0722ce59a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Thu, 7 Nov 2019 11:56:10 +0100 Subject: [PATCH 040/257] cmd/thanos/receive: avoid deadlock (#1727) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While debugging #1721, I found that when thanos receive bails, there is a race in a select statement, where the non-returning branch may be chosen. This branch will deadlock if selected twice because the channel reader has already exited. The way to prevent this is by checking if we need to exit on every loop. Signed-off-by: Lucas Servén Marín --- cmd/thanos/receive.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 1424a13506..44340718bd 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -248,7 +248,6 @@ func runReceive( defer func() { if err := db.Flush(); err != nil { level.Warn(logger).Log("err", err, "msg", "failed to flush storage") - return } if err := db.Close(); err != nil { level.Warn(logger).Log("err", err, "msg", "failed to close storage") @@ -449,6 +448,11 @@ func runReceive( }() defer close(uploadDone) for { + select { + case <-ctx.Done(): + return nil + default: + } select { case <-ctx.Done(): return nil From 4b325bd1c0f04528ffd209108aa1094e82c5b658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Thu, 7 Nov 2019 11:57:16 +0100 Subject: [PATCH 041/257] cmd/thanos/receive: reduce WAL replays at startup (#1721) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every time thanos receive is started, it has to replay the WAL three times, namely: 1. open the TSDB; 2. close the TSDB; open the ReadOnly TSDB and Flush; and 3. open the TSDB These WAL replays can take a very long time if the WAL has lots of data. With the fix from #1654, the third time will be instantaneous because the WAL will be empty. That still leaves two potentially long WAL replays. We can cut this down to just one long replay if we do the following operations instead: 1. with a closed TSDB, open the ReadOnly TSDB and Flush; and 2. open the TSDB Now, the second step will be a fast replay because the WAL is empty, leaving just one potentially expensive WAL replay. This commit eliminates explicit opening of the writable TSDB during startup, and instead opens it after flushing the read-only TSDB. Signed-off-by: Lucas Servén Marín --- cmd/thanos/receive.go | 17 ++++++----------- pkg/receive/tsdb.go | 25 +++++++++++++------------ 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 44340718bd..18d4474066 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -230,6 +230,9 @@ func runReceive( defer close(dbReady) defer close(uploadC) + // Before actually starting, we need to make sure the + // WAL is flushed. The WAL is flushed after the + // hashring is loaded. db := receive.NewFlushableStorage( dataDir, log.With(logger, "component", "tsdb"), @@ -237,22 +240,11 @@ func runReceive( tsdbCfg, ) - // Before actually starting, we need to make sure the - // WAL is flushed. The WAL is flushed after the - // hashring ring is loaded. - if err := db.Open(); err != nil { - return errors.Wrap(err, "opening storage") - } - // Before quitting, ensure the WAL is flushed and the DB is closed. defer func() { if err := db.Flush(); err != nil { level.Warn(logger).Log("err", err, "msg", "failed to flush storage") } - if err := db.Close(); err != nil { - level.Warn(logger).Log("err", err, "msg", "failed to close storage") - return - } }() for { @@ -266,6 +258,9 @@ func runReceive( if err := db.Flush(); err != nil { return errors.Wrap(err, "flushing storage") } + if err := db.Open(); err != nil { + return errors.Wrap(err, "opening storage") + } if upload { uploadC <- struct{}{} <-uploadDone diff --git a/pkg/receive/tsdb.go b/pkg/receive/tsdb.go index 16fdae6e8f..da463fe874 100644 --- a/pkg/receive/tsdb.go +++ b/pkg/receive/tsdb.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/storage/tsdb" @@ -69,32 +70,32 @@ func (f *FlushableStorage) open() error { } // Flush temporarily stops the storage and flushes the WAL to blocks. -// Note: this operation leaves the storage in the same state it was in. +// Note: this operation leaves the storage closed. func (f *FlushableStorage) Flush() error { + _, err := os.Stat(filepath.Join(f.path, "wal")) + if os.IsNotExist(err) { + level.Info(f.l).Log("msg", "No WAL was found for flushing; ignoring.") + return nil + } + if err != nil { + return errors.Wrap(err, "stating WAL") + } f.mu.Lock() defer f.mu.Unlock() - var reopen bool if !f.stopped { if err := f.DB.Close(); err != nil { return errors.Wrap(err, "stopping storage") } f.stopped = true - reopen = true } - ro, err := promtsdb.OpenDBReadOnly(f.Dir(), f.l) + ro, err := promtsdb.OpenDBReadOnly(f.path, f.l) if err != nil { return errors.Wrap(err, "opening read-only DB") } - if err := ro.FlushWAL(f.Dir()); err != nil { + if err := ro.FlushWAL(f.path); err != nil { return errors.Wrap(err, "flushing WAL") } - if err := os.RemoveAll(filepath.Join(f.Dir(), "wal")); err != nil { - return errors.Wrap(err, "removing stale WAL") - } - if reopen { - return errors.Wrap(f.open(), "re-starting storage") - } - return nil + return errors.Wrap(os.RemoveAll(filepath.Join(f.path, "wal")), "removing stale WAL") } // Close stops the storage. From 1acdf7cd954352e89792404ad602fb11188ad164 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 8 Nov 2019 09:45:43 +0000 Subject: [PATCH 042/257] Added Lucas to mantainers list. (#1732) Signed-off-by: Bartek Plotka --- MAINTAINERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index a76fbe3b21..8f0bc525f2 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -8,6 +8,7 @@ | Giedrius Statkevičius | giedriuswork@gmail.com | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS) | AdForm | | Povilas Versockas | p.versockas@gmail.com | `@povilasv` | [@povilasv](https://github.com/povilasv) | Utility Warehouse | | Matthias Loibl | mail@matthiasloibl.com | `@metalmatze` | [@metalmatze](https://github.com/metalmatze)| Red Hat | +| Lucas Servén Marín | lserven@gmail.com | `@squat` | [@squat](https://github.com/squat) | Red Hat | We are bunch of people from different companies with various interests and skills. We are from different parts of Europe: Germany, Lithuania, Poland and UK. From 3debaeba1a0ec2eee771b801d887c1597dcf0ced Mon Sep 17 00:00:00 2001 From: Alfred Landrum Date: Sat, 9 Nov 2019 15:47:42 +0100 Subject: [PATCH 043/257] store the first raw value of a chunk during downsampling (#1709) * store the first raw value of a chunk during downsampling As discussed in #1568, storing only the last raw value of a chunk will lose a counter reset when: a) the reset occurs at a chunk boundary, and b) the last raw value of the earlier chunk is less than the first aggregated value of the later chunk. This commit stores the first raw value of a chunk during the initial raw aggregation, and retains it during subsequent aggregations. This is similar to the existing handling for the last raw value of a chunk. With this change, when counterSeriesIterator iterates over a chunk boundary, it will see the last raw value of the earlier chunk, then the first raw value of the later chunk, and then the first aggregated value of the later chunk. The first raw value will always be less than or equal to the first aggregated value, so the only difference in counterSeriesIterator's output will be the possible detection of a reset and an extra sample after the chunk boundary. Fixes: https://github.com/thanos-io/thanos/issues/1568 Signed-off-by: Alfred Landrum * changelog for #1709 Signed-off-by: Alfred Landrum * adjust existing downsampling tests Signed-off-by: Alfred Landrum * add counter aggregation comments to CounterSeriesIterator Signed-off-by: Alfred Landrum --- CHANGELOG.md | 1 + pkg/compact/downsample/downsample.go | 77 +++++++++----- pkg/compact/downsample/downsample_test.go | 122 +++++++++++++++++++++- 3 files changed, 169 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8f4785457..8945f3425a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1656](https://github.com/thanos-io/thanos/pull/1656) Thanos Store now starts metric and status probe HTTP server earlier in its start-up sequence. `/-/healthy` endpoint now starts to respond with success earlier. `/metrics` endpoint starts serving metrics earlier as well. Make sure to point your readiness probes to the `/-/ready` endpoint rather than `/metrics`. - [#1669](https://github.com/thanos-io/thanos/pull/1669) Fixed store sharding. Now it does not load excluded meta.jsons and load/fetch index-cache.json files. - [#1670](https://github.com/thanos-io/thanos/pull/1670) Fixed un-ordered blocks upload. Sidecar now uploads the oldest blocks first. +- [#1568](https://github.com/thanos-io/thanos/pull/1709) Thanos Store now retains the first raw value of a chunk during downsampling to avoid losing some counter resets that occur on an aggregation boundary. ### Changed diff --git a/pkg/compact/downsample/downsample.go b/pkg/compact/downsample/downsample.go index 05b1c1e46d..4f223064c6 100644 --- a/pkg/compact/downsample/downsample.go +++ b/pkg/compact/downsample/downsample.go @@ -289,10 +289,6 @@ func (b *aggrChunkBuilder) add(t int64, aggr *aggregator) { b.added++ } -func (b *aggrChunkBuilder) finalizeChunk(lastT int64, trueSample float64) { - b.apps[AggrCounter].Append(lastT, trueSample) -} - func (b *aggrChunkBuilder) encode() chunks.Meta { return chunks.Meta{ MinTime: b.mint, @@ -306,14 +302,17 @@ func downsampleRaw(data []sample, resolution int64) []chunks.Meta { if len(data) == 0 { return nil } - var ( - mint, maxt = data[0].t, data[len(data)-1].t - // We assume a raw resolution of 1 minute. In practice it will often be lower - // but this is sufficient for our heuristic to produce well-sized chunks. - numChunks = targetChunkCount(mint, maxt, 1*60*1000, resolution, len(data)) - chks = make([]chunks.Meta, 0, numChunks) - batchSize = (len(data) / numChunks) + 1 - ) + + mint, maxt := data[0].t, data[len(data)-1].t + // We assume a raw resolution of 1 minute. In practice it will often be lower + // but this is sufficient for our heuristic to produce well-sized chunks. + numChunks := targetChunkCount(mint, maxt, 1*60*1000, resolution, len(data)) + return downsampleRawLoop(data, resolution, numChunks) +} + +func downsampleRawLoop(data []sample, resolution int64, numChunks int) []chunks.Meta { + batchSize := (len(data) / numChunks) + 1 + chks := make([]chunks.Meta, 0, numChunks) for len(data) > 0 { j := batchSize @@ -327,14 +326,18 @@ func downsampleRaw(data []sample, resolution int64) []chunks.Meta { for ; j < len(data) && data[j].t <= curW; j++ { } - ab := newAggrChunkBuilder() batch := data[:j] data = data[j:] + ab := newAggrChunkBuilder() + + // Encode first raw value; see CounterSeriesIterator. + ab.apps[AggrCounter].Append(batch[0].t, batch[0].v) + lastT := downsampleBatch(batch, resolution, ab.add) - // InjectThanosMeta the chunk's counter aggregate with the last true sample. - ab.finalizeChunk(lastT, batch[len(batch)-1].v) + // Encode last raw value; see CounterSeriesIterator. + ab.apps[AggrCounter].Append(lastT, batch[len(batch)-1].v) chks = append(chks, ab.encode()) } @@ -379,18 +382,21 @@ func downsampleBatch(data []sample, resolution int64, add func(int64, *aggregato // downsampleAggr downsamples a sequence of aggregation chunks to the given resolution. func downsampleAggr(chks []*AggrChunk, buf *[]sample, mint, maxt, inRes, outRes int64) ([]chunks.Meta, error) { - // We downsample aggregates only along chunk boundaries. This is required for counters - // to be downsampled correctly since a chunks' last counter value is the true last value - // of the original series. We need to preserve it even across multiple aggregation iterations. var numSamples int for _, c := range chks { numSamples += c.NumSamples() } - var ( - numChunks = targetChunkCount(mint, maxt, inRes, outRes, numSamples) - res = make([]chunks.Meta, 0, numChunks) - batchSize = len(chks) / numChunks - ) + numChunks := targetChunkCount(mint, maxt, inRes, outRes, numSamples) + return downsampleAggrLoop(chks, buf, outRes, numChunks) +} + +func downsampleAggrLoop(chks []*AggrChunk, buf *[]sample, resolution int64, numChunks int) ([]chunks.Meta, error) { + // We downsample aggregates only along chunk boundaries. This is required + // for counters to be downsampled correctly since a chunk's first and last + // counter values are the true values of the original series. We need + // to preserve them even across multiple aggregation iterations. + res := make([]chunks.Meta, 0, numChunks) + batchSize := len(chks) / numChunks for len(chks) > 0 { j := batchSize @@ -400,12 +406,13 @@ func downsampleAggr(chks []*AggrChunk, buf *[]sample, mint, maxt, inRes, outRes part := chks[:j] chks = chks[j:] - chk, err := downsampleAggrBatch(part, buf, outRes) + chk, err := downsampleAggrBatch(part, buf, resolution) if err != nil { return nil, err } res = append(res, chk) } + return res, nil } @@ -512,6 +519,9 @@ func downsampleAggrBatch(chks []*AggrChunk, buf *[]sample, resolution int64) (ch ab.chunks[AggrCounter] = chunkenc.NewXORChunk() ab.apps[AggrCounter], _ = ab.chunks[AggrCounter].Appender() + // Retain first raw value; see CounterSeriesIterator. + ab.apps[AggrCounter].Append((*buf)[0].t, (*buf)[0].v) + lastT := downsampleBatch(*buf, resolution, func(t int64, a *aggregator) { if t < mint { mint = t @@ -520,6 +530,8 @@ func downsampleAggrBatch(chks []*AggrChunk, buf *[]sample, resolution int64) (ch } ab.apps[AggrCounter].Append(t, a.counter) }) + + // Retain last raw value; see CounterSeriesIterator. ab.apps[AggrCounter].Append(lastT, it.lastV) ab.mint = mint @@ -532,11 +544,18 @@ type sample struct { v float64 } -// CounterSeriesIterator iterates over an ordered sequence of chunks and treats decreasing -// values as counter reset. -// Additionally, it can deal with downsampled counter chunks, which set the last value of a chunk -// to the original last value. The last value can be detected by checking whether the timestamp -// did not increase w.r.t to the previous sample. +// CounterSeriesIterator generates monotonically increasing values by iterating +// over an ordered sequence of chunks, which should be raw or aggregated chunks +// of counter values. The generated samples can be used by PromQL functions +// like 'rate' that calculate differences between counter values. +// +// Counter aggregation chunks must have the first and last values from their +// original raw series: the first raw value should be the first value encoded +// in the chunk, and the last raw value is encoded by the duplication of the +// previous sample's timestamp. As iteration occurs between chunks, the +// comparison between the last raw value of the earlier chunk and the first raw +// value of the later chunk ensures that counter resets between chunks are +// recognized and that the correct value delta is calculated. type CounterSeriesIterator struct { chks []chunkenc.Iterator i int // Current chunk. diff --git a/pkg/compact/downsample/downsample_test.go b/pkg/compact/downsample/downsample_test.go index b5e6a40634..3c0fce8851 100644 --- a/pkg/compact/downsample/downsample_test.go +++ b/pkg/compact/downsample/downsample_test.go @@ -23,6 +23,124 @@ import ( "github.com/thanos-io/thanos/pkg/testutil" ) +func TestDownsampleCounterBoundaryReset(t *testing.T) { + + toAggrChunks := func(t *testing.T, cm []chunks.Meta) (res []*AggrChunk) { + for i := range cm { + achk, ok := cm[i].Chunk.(*AggrChunk) + testutil.Assert(t, ok, "expected *AggrChunk") + res = append(res, achk) + } + return + } + + counterSamples := func(t *testing.T, achks []*AggrChunk) (res []sample) { + for _, achk := range achks { + chk, err := achk.Get(AggrCounter) + testutil.Ok(t, err) + + iter := chk.Iterator(nil) + for iter.Next() { + t, v := iter.At() + res = append(res, sample{t, v}) + } + } + return + } + + counterIterate := func(t *testing.T, achks []*AggrChunk) (res []sample) { + var iters []chunkenc.Iterator + for _, achk := range achks { + chk, err := achk.Get(AggrCounter) + testutil.Ok(t, err) + iters = append(iters, chk.Iterator(nil)) + } + + citer := NewCounterSeriesIterator(iters...) + for citer.Next() { + t, v := citer.At() + res = append(res, sample{t: t, v: v}) + } + return + } + + type test struct { + raw []sample + rawAggrResolution int64 + expectedRawAggrChunks int + rawCounterSamples []sample + rawCounterIterate []sample + aggrAggrResolution int64 + aggrChunks int + aggrCounterSamples []sample + aggrCounterIterate []sample + } + + tests := []test{ + { + // In this test case, counter resets occur at the + // boundaries between the t=49,t=99 and t=99,t=149 + // windows, and the values in the t=49, t=99, and + // t=149 windows are high enough that the resets + // will only be accounted for if the first raw value + // of a chunk is maintained during aggregation. + // See #1568 for more details. + raw: []sample{ + {t: 10, v: 1}, {t: 20, v: 3}, {t: 30, v: 5}, + {t: 50, v: 1}, {t: 60, v: 8}, {t: 70, v: 10}, + {t: 120, v: 1}, {t: 130, v: 18}, {t: 140, v: 20}, + {t: 160, v: 21}, {t: 170, v: 38}, {t: 180, v: 40}, + }, + rawAggrResolution: 50, + expectedRawAggrChunks: 4, + rawCounterSamples: []sample{ + {t: 10, v: 1}, {t: 30, v: 5}, {t: 30, v: 5}, + {t: 50, v: 1}, {t: 70, v: 10}, {t: 70, v: 10}, + {t: 120, v: 1}, {t: 140, v: 20}, {t: 140, v: 20}, + {t: 160, v: 21}, {t: 180, v: 40}, {t: 180, v: 40}, + }, + rawCounterIterate: []sample{ + {t: 10, v: 1}, {t: 30, v: 5}, + {t: 50, v: 6}, {t: 70, v: 15}, + {t: 120, v: 16}, {t: 140, v: 35}, + {t: 160, v: 36}, {t: 180, v: 55}, + }, + aggrAggrResolution: 2 * 50, + aggrChunks: 2, + aggrCounterSamples: []sample{ + {t: 10, v: 1}, {t: 70, v: 15}, {t: 70, v: 10}, + {t: 120, v: 1}, {t: 180, v: 40}, {t: 180, v: 40}, + }, + aggrCounterIterate: []sample{ + {t: 10, v: 1}, {t: 70, v: 15}, + {t: 120, v: 16}, {t: 180, v: 55}, + }, + }, + } + + doTest := func(t *testing.T, test *test) { + // Asking for more chunks than raw samples ensures that downsampleRawLoop + // will create chunks with samples from a single window. + cm := downsampleRawLoop(test.raw, test.rawAggrResolution, len(test.raw)+1) + testutil.Equals(t, test.expectedRawAggrChunks, len(cm)) + + rawAggrChunks := toAggrChunks(t, cm) + testutil.Equals(t, test.rawCounterSamples, counterSamples(t, rawAggrChunks)) + testutil.Equals(t, test.rawCounterIterate, counterIterate(t, rawAggrChunks)) + + var buf []sample + acm, err := downsampleAggrLoop(rawAggrChunks, &buf, test.aggrAggrResolution, test.aggrChunks) + testutil.Ok(t, err) + testutil.Equals(t, test.aggrChunks, len(acm)) + + aggrAggrChunks := toAggrChunks(t, acm) + testutil.Equals(t, test.aggrCounterSamples, counterSamples(t, aggrAggrChunks)) + testutil.Equals(t, test.aggrCounterIterate, counterIterate(t, aggrAggrChunks)) + } + + doTest(t, &tests[0]) +} + func TestExpandChunkIterator(t *testing.T) { // Validate that expanding the chunk iterator filters out-of-order samples // and staleness markers. @@ -56,7 +174,7 @@ func TestDownsampleRaw(t *testing.T) { AggrSum: {{99, 7}, {199, 17}, {250, 1}}, AggrMin: {{99, 1}, {199, 2}, {250, 1}}, AggrMax: {{99, 3}, {199, 10}, {250, 1}}, - AggrCounter: {{99, 4}, {199, 13}, {250, 14}, {250, 1}}, + AggrCounter: {{20, 1}, {99, 4}, {199, 13}, {250, 14}, {250, 1}}, }, }, } @@ -93,7 +211,7 @@ func TestDownsampleAggr(t *testing.T) { AggrSum: {{499, 29}, {999, 100}}, AggrMin: {{499, -3}, {999, 0}}, AggrMax: {{499, 10}, {999, 100}}, - AggrCounter: {{499, 210}, {999, 320}, {1299, 430}, {1299, 110}}, + AggrCounter: {{99, 100}, {499, 210}, {999, 320}, {1299, 430}, {1299, 110}}, }, }, } From f86cf8178a7ee3d8ee54c481232ad8dc794db228 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 11 Nov 2019 13:20:05 +0100 Subject: [PATCH 044/257] Adds FREE NOW as users of Thanos (#1738) Signed-off-by: Sebastian Herzberg --- website/data/adopters.yml | 3 +++ website/static/logos/free-now.png | Bin 0 -> 19577 bytes 2 files changed, 3 insertions(+) create mode 100644 website/static/logos/free-now.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index 8eac02bc75..9e8a71b705 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -57,3 +57,6 @@ adopters: - name: BlaBlaCar url: https://www.blablacar.com logo: blablacar.png +- name: FREE NOW + url: https://free-now.com + logo: free-now.png diff --git a/website/static/logos/free-now.png b/website/static/logos/free-now.png new file mode 100644 index 0000000000000000000000000000000000000000..2115fd4ba173f72491f016b533b4775a5a152bf7 GIT binary patch literal 19577 zcmZr&c|6qJ_y0(RtnrlWR7hEhBs)=}&~6=)O4;|_j5Ycyl&usaq{W(jACbhwknF~g zG-DkMW*BCEchK|v^Xs2pGxNFUp7njsJ?DNrzG7m)xleE(1VNmJ7xk|}5K9*XF+E`2 z3%=p@xNQag6YxH7?tR_uzPF#9=RHXGuG^h^;)X7E4)?Czv%4GM(RNP*f-Vag>g(R{ zADYE#dkGoVNKgM5Ipx}A(II{If!{-pVO-3kx*6pIx)Xunbt^{$mUC9e4A*x@2J6Qt z7D;-4S#Lg7=4!9?*7~bGV1b;TQ_NnxM?-O&j0r> zW&Yu=t?vV;2$aoVm^Y6og}9n2O4NPm!Fr%SV$q>mN_4qSvXdW)+(X!Ua=E{IF}4MX zQU2oBaS$1e^$sqSfzoQZ_F?iop9U3I^`*0QF)^MbRAx_%|H4$pVnUuS8cUiVlUToe z9Xjo`$(LbkJ0agpTB1{8Z5jH4p!!>SsHW(g zpz6Y{Ks7Jaq#^qt6)*To<-~H(Cl)Wq(;##E(|-< zw%SJ`pU>wVvpdh$ZK}Md(WLMuAUxCEJ|6HUQhkf?_5+_ghUW>Cx4F=O;`my!Q1pMo zTLCe$cJp&5d$SC*`xyG$V2^7k?B3A~|Bj3uw(*hp{{GiifHB_rExPW1e>3LIu&uuC zZ!-|Kt+Sb|9P05TZx(Z6xdEW2`Y*L&lO1X+)?N&1$?AZO9cusnRtul2#Z3|2Q_q<$m5Des5wtz>Z~G^|SxmUPB=H zob$JFk^GOfOWUkv#`m*~udOKGKJlM9W}ojY;s*pby@|0r&yig|e!dLkC z8W&K}EwaMMwZAUu%q#qdg)SB+kDB~Tad@}(` zF$N(UhJD)8L>l{#*sW~~hq=UeG?9)6{~LRmlG`@9ZDn7wun{;Cl3=&^d`B9~>>TPx zsr2XbjrUvcj0a>b)bVV`vQFMMYmHK0ZN~$W{|k&R#cc;G?({GN=BUv6R-t3KqwRw0 zKSFLMni%?x%}*38XgvIn1%(}IF0JN(+L?c~)+G&i`#7tQcq{Ga{ zeQ#Xx$!Q4bUU(dPe%RsttD{s%Zgf{QaC`d@%F&TaD| z(fUG4gXr-erP>`zv!5>ekCOhQH1lSM5|76QZw5O?VbivAI}_cOqmk)pKJ-n*Mgyg? zxdl(}6SuwTJ445uSI5>mE8Hdn6)Xg|aczhG?Rucz-ux}wM`NFwHF=G%IqX1l<9$Y& z;k8cY3CWeYJ!7A=VC_$nHmcgz@hb)kMzv3pyrbp#GWeTFMa9h9kvouY`D1hQmsjSg z2B$t?>)}I*k@zN(8}GKqL&F~48=i~!>zs=)qnhUXtrFwa3mdgb?puHWRZJl0Mf4&n zkv@TpMNknS0#WA;{RaL4zM?^AQs2&Vp>t|#qO!E$VVNn~ky79V=KWPiH25=$^mjsJ z_R%&>%xm?rVxsUTuJ0i20OxjCjYI)$X5d2qdV?#lz(+<(9l7hgA86F=@(!UdnOhJM z-TRe~!RM{NeBRl{r{;JNE8}Z&gFC6|3rHp(?*khzOj0BOL%7E~2oSRa@%{FXfF9uv zfAx3@Yz*?n8y#=V+mfyx)BqhVC#ncI|wH(4ODS;S4AtRfj-JSAdK(>xm7Og zR14d`4o%WwWj&kd)$rLZzloULMpVdiJHit;t+!U0>}acOhrK?MmA||OZr@HN6MAMF zGuuGAX2n!~+3~(iIe-U7hJK2#F$~AMwS(iKXSQ|nIGBqa*v{(RPMy&uwQajoYz&oi z{jCPd7`l>~GD-5f#n@9|r)=rs2E;-2M7V7#`BVTK?3m{LJ3jx4K|9Q8QC|QF`^~l+Fu5TK{-ML(B{)b z`niKTP$i&Xy@e>h|9<5Ms#aDEC~=toelwGqqaGruK~x(LQ2O_~Eb!eC4TVTN=U=tw zgPLo3E7Dai>?pIC&8VhFaXg@(4fE>gjx!JZBZ+gzwO>;!_4Oa^&BY{9q;X*25x!bBr5-gBTagZH=)z`|~4iq3P!hH~tf^lD5 zE1E>v?!gah#4S0z%si3kh4SlXs&yBELS1l27$BAZY>SV_t@ zHhxv%Vd;%j{4b@M+=-W+yP?~9UC#GwVnl2jfKPOr{v-I$-|0W zR#r;X!>+-9s zGt84K@9HkO_w{Cej?n9ZN2tG$!>q{eh12sFE42-2B4?=1?%o^ZmWVlf?c|?_ab~D4 z&9S7sQ7^JL?kb}IAl zqQgQVO&@RT@J?R$jiKJCL*$?#-sX8n2_v~Rcnzg)e=g4QAu5#(ZaV2lT(3k2C3I8^ z4lfg33*{o@;r^p*A&`?WsF9ky&JX}pvSw*K;HnjTkAel$LOxZ}<6H2Pzr3m_BmMeN zS5o3Vwwsv>?It9(ZZffO9@3m$7s^09?CMRQqMMg>i^XElEqI;M5uNn2n#S%q(dhF> zXTN51(#gmNzeF;uRO`BLHVV$?MKN{*^@hIh&C!M9R}faC_6nC-Lhrw3jmh-5XK}xELrk}+RvNn0`ZTqe{A_YibvQwRyB>>pLpH85UOmA zVTe!xFzN0!DFJ!EeAXVOOaWpPp(|5xy}@>4SPaG)fvv=Ke(qK3n!NjAafzBcVPUrG zBW3p5=u?CB%0Z{j+p0Df$d(RFe!|2pVgL~C`W(omdwi-ylu%w=+!gxT=>2NnywL+4 z)%f^0Tk90@BJ#A}(~?`+h4V%T;(-wx}~IZ zSotg}e_yXmPXl{|NdnD5IWtZ2?6rFh!>&Ywv|y+HtO)tmrUE}T_NvLV(hRl1J#Z`gKKWgJZxlDJ674=U1%Sh| z_W}1R}LMfi9MN%%P$nW~U@9p3lqxsxle zHw?^h9)&l8oF`x6=dv)PWZ!77^t&BA<^kO6e_9Kv>dp@AN6+9;(T0VFMLq}Jn-`54 zLEXBWpo2_e$tw}Ux(Yxs#4lV%teLxuC1d39cjNrNd%BosAmq_={XC!CN|HSM3Whf{ z8bdaR^KJQrC+c3sVSs=UR5wN-LVyx3i3Oj^A4Db2U0?}aFsTze==;0II%v>G>!V71)sJWEx-A+xzn)=`5(lJgIcsZ#LULpv z@qEQomPS0>r}sE_cP~!Xdzzukdpi9CZ}!!j+B-fPYv-EmczW;N{?X5!)c%m~QkZm( zUS+A8eddcblE8L+Ad3)6moj}kKULaljZUzFMp(aFkR|+3Mc%v`VlX-CM01CiwX(k$lb@V$GWs z4N}mL=KQFxj4mX-6U+`sEh@KM4xq*+I=Uy1W1HKn+~zH`!=-EP%=ODo4u_?DJ z6jUfTU!L5UxZ{Qky2T_uU%vkKekf-Zynju{e%Xi(Ni$EGbsL=dC&cdAKuv4;RsMH% zmo)BhJz$-T^o#^QU29FC)xIoXY+K_!JzfbIqb z5T%AsLJf*s@w7-gQ1vvBc5JXS^$|>NYV?;=!KmcfuMzi7e#4%JJsBIlr>S_u=>=#n zUN&RGZL?wsplQK{q1E7Y*VDhlIzICG+C(*0CW(~LAGkjI;ceYsh2aBC%)3-(A7|ct zXh76rY4B6fnhrR@iZRp$t3$x*n3;MWkz3xI4PNAm{AJ-XN`0O4tol9HvfcT9Vlzhm zTC9QI%w##gd0d3N!AerU8at)^5CeP?fuoGI+5klaecKc2-)_=I9+lareFgK3Hv{NG zWx-pud-#7GVJV?2^P+pSwPKGB&EtOMzly`x?4$gGTZ3ql1}BZic$mpqKm_WXSH=WOmf- zYdw~h&KeVHI^W9hab8eIj(jJ8S|mpR693~=%e*f4|n2Ys^~SgU4r zP}R<{Ty-N$FdrXy?p-7)BjpOx)H5_V7h(J(p znTdv)t4;P57w@dhSP3{N%03CsF3luQBm-nN(F4I)N!|dVxLXqXa-*;tB*o?Mh1b|` z!dWO2_b1V9mnPkM5_4J$o=`vH&jmYx({XNIWN%elKO~O^Y7nH1ei$&WRujMi^YdEw;&^O479C<={nN1D~}vGe35#nqVo(& zoQztvDzO{XFf&17+U5^Api{89gyoZ_K@6j>spBz!kioQ-7CJ7Tqh{vHJP`)Vbnmiw zzo6WVempTn@hF*qCytHCsOL$;@8FU{`2D6P9BLL{^;au$IhbGkr4dDoV;l!np90Vr z!J^Txc0=MHc@vTzKaEZc-%Hes!h^<7q91+MNC&|5nN<}S(O>TK`V}#2LAjwxWUMbD zneajyw(-9^EU8R)u%|FcOp7e>D~(otM2DWiu^Dvyboi)EG{XS|SNrv)-|!{$ZKiBc zihS?=6Cv-$Rz7@)5gEjgjgus$(f1^IgO}&Eh)N^uE9Iyd{K+XULrP8d1V0QVJ=15{~XtS*f7J%y^^6Zv6#rmB#F%a;+*_sJ`S+r`N-Cykf;NQzsG`F2=oao)3EE zNt}odU<+>l721LaMR;Wkos5CWnf49M*NuCAh9DlRK?&6#963JT!7E1L4aYuznQ&F3 z4d2`InD>T0^8Ja)igy!jzpmQ|E%q#}IgBRKYH+-Ns$DGw}=tOhNTgP(xnS8$7?^xG19=`111c6H`{z16uvfP(o8%$%x|b zYkF;p5B*S!Zk8F`@Y-OTV-ntM*`*)t*7I84X;70KjS@vSu9eCyee05rS8G_#`9xIu z@fPv?O?dRuTvt($2P3hGf@n}L9eHEpYoFAXv!wN7mVdN4wtkZuco4eg?_?mxRzEug zN3GV?j=Io(DF(Y5%8+b)yDKXbD0!F*v8PraKDQ4^%+h>VZ!G(qa--frC=zvlG61&6 zo$SHrR=fkP$i_}Al`* z@qAUhi0JMm=QzMxsQRY6bOg_Nd%h=r4H+C$IlGC;JA%i`#xQC;TUS^yI453_dL}BZ;ms!mC7?fhdhY)9f%r`H%<|Lrqa&D9$?wKrLJE_3 zk#Sx=DRP>L@kp944m5|r=(lmAK?haf8mZ&Llf)Y#zS5~em@fcwruGPJ9TeeaWq1OfDPvaCtQn70Va~CjI+F=4@+5VY| z6>7P=&+C{5nQ`lte;&YP8SQA0oxh}>)E5Kcf1-9Tw5CyB8-LqU1`98r*@%*dU6q@G zmkZVJiI88vlQ0GHL%fz@@W2c&ZbcBgP8N$S8YrH^`9E5k(?RzO0-XxW1JVCUkyt<4 zEy2Aa2<3Pm)|ZZ4S9c!IC>d$iE}q#)DfG+62vN@5d-g*&9OvX60>5W&>pQ0XqxDU_ zuSfWOB8rjcDBMhH3164e2^LbL@NRo3p%1-Vf?F<6{j!z+Tb%&RnmwPp-wl4TB%b;x zSiJP-cm+KHGCYM+SX4$p{|bZO68v_xA;eYp&}sF(j3Q_O6aWdvDc9zuuuS4bieXfn zm`*NDqGzg%e4;CNO6@y~XvJr$BHGp)R;y<;Sz(jed+kZPUCt85Ely5%`*rXfbvyTn zxRJ{|wAJxg*aOJ5*T;*sw`erybYF2}(s8`Ew4T#XOuMJzcyjghqDs|1jDwmNbMC~F z$#QEOss%k}@APQtdkBXa^;Tb9eDYq!VEkkD{ipqiasUdO&4AvXz-OkvSU&)%(zk#~ z$92+@2DnT~KdQB=xVDOZ9?;HY=2aS&5xSXjCR{rxekp|G7xw~Q&ZeIol z2t)2wuWyGL2fbXbGH#eVhn=ElqJ^_sSLAi6D@Se=-n`8f6qq8m`hK6ua?37@GoDZ$ z%qkK;j~kB$rwAjm7W{>R!t%4t)QnxB_n$D*OIp}dRW(r`RsP#)!NYF_v$QX36mo@E z%;dzPPQFoiv-G~z!C(6z_fN-t5n~1c(`qaoPgI+{$i!mMuS&33hA}b2c0Z@n1LLR4{qg2a*IC5=y9s%Uj$N}VTT7Z z)_15{G`th+dnf<>n=jE^3M(p$mw2e@>e9LyxscaDWo%2R=n_2|Z1TTA(YpfyLyiJuy?0B}WZ?q&q!=8=!RF?!$84CH zvr-z-Rv>Q-Zi=prF_Lo)E0lJ#y$aROF=82Vpz&n){?x~n(Q5KlYPx9!NG?73iyuD^ zeyA9JEN$`!Q&xH97ojQ=-;JmRrPSytMqfHw9guVG)8T`3dbm;&mE45}cV$IOjXi%6 z=Kzx_M;iS0#1eVSY^`d`XMSZdOE2+PHsymPCI?5D(=G% zN~Agpv_C&DSB{(tA()0&OytDc@}kPIL%vAF3_QEQ>GpDz7~f~U9AR6AUd^|zN;G1h z3c|HzUafchK1KALyZ>isLcdkbJjFZi*t-1Bi2J_Mc-C8flUMa{l6qn&QeuR6KMw=l z`nFor#iG*Zyj{h6xX=z#H*pYkT_wFc7i?x>kL$ku&doyggftw| zPIimF%oxcC_W?&MUK8SnGPP~waoByhTZkhOM-9~Z4j(nUREk++gdRA)h$SYP)`Na^QA_a4*eHl}>VlY|~SAUCzLSx9%ZbC1{ z?t7IajfM0Yw|ZfFL#O=vk{x%c>SOKHyM2YoU)pM(UtOArr>SdN-AaeD_%ixWUGnD= z0)6QcA;(>ic&4tIUlx1!nso@m}dHuMz@=`DNxYf=QTSZlk z&7S+c(bcTXHZv-w3VS};a#ZIeNaW(o9XIWDz0?C}sLO5AW>Q~iA<8yU81Izm@$`fG zacRT2Gmbp3-#iKG8DgIm26{z?D} zz3S(^aZagtw2loT|^C;bp-jIOy$p!ZD5 z-?O~w_w&C0tu@I9)Z~g-c--9+6QZZg(-FM67v!)=a3nfLSv0m57eB_BFo3~tCbPvo z|0}O&03mj1G;i**vD{USi|#qTNV47|#g^zHB9}|*+d4m(`9Tloy3xnNmELsSw4nbD zVY(5C?Uj1p7xfqsoDQaKQ`WO{D8esOtqdHMVpmuR-JX+bi$`4+MRBjW+MaQFF?I-# z#+H|XmY=u4!3|Ln15GzY>)4VT8jowv($FpXSDSmK4cAozZ$_?_sIlj4>>n^Z&ZcNI9DSBUfuT-pIN#D@(#7oReRc_3)& z6)(F#;lutsA+Rg^8@p+i!dcaeSLS}7w7^}^a8Qiw5ooV;Fu$^U6O4J(##LuQ<#iEs zhUgDd-*I3s$1^oP`e_&$9c#MU7Rj51`uRpPVPX^SfEe;HoUxy(i$xuJc!)A!!eOvAD2%u6q?%)-Hw>ncNlz} z5pvD0?Df$sm*(BA?}LN1TZrWMMBSpqp>DqdEw<632;hI~9gT-wk*x)160(D->Z!X} zmF_bFoaYfJ6Ws(d9Ma5ht+PWFc53p8?A$^WDM@{?sjl2jtfuVxW*^rsF{|Z0tPw3M zl8#i(eSy|RiK_uWTX?&hNu-v|-(TKByBO1MFILb|j!ZeJp^$I0H`L{2oI|X(n972& zNhHBoofRJJHR%+j{-{%ds$F5bs}d)xOKoBQI8*pLroV2@(_@spzH#~i(x5&%#>Qs! z*Qpo}7!Lq)31Lvs_qB||Wd<`o<$Pc{2@T}dco8g^alel)nCQ;mC(sUTBAl4#{G|8$ zLGj@gy)y^s6#SJ+!h9CwsP9VG^h{c@U z=t>OM?QA+9;q9{rFvt zCduSC4mjh}glUsyYE3CyH-9AZWm$GCD&`K)1|=&sEUv6r*71E{Ps_s9=KiZO+3(pU zvyhd1dfKzjPqwIQFsT(Q0WC+zVulO%;_u-DRt@iP+d{=oKD)B&?jH_{v60HaO?OFc zXi^qsY%e{+eHN!Io=0#8J|Ww;7Vs*eJRVc0sx;^Z9Gar4amn9z&Hapq@6WKs`mtdo z0D2cIqtpLn^QI|i(dVag7VPwF7t4XqLDTyBjCeSA2Na!$D+#gs^bfIw>7cG7RLRdi z4!m`o2u^p7Xx-K?!}C({1NfD6@~L$-#mQki{w-{O4q(g44*+p7g@YgY?(J#nFA`kf zMQZvV=`gW0;vGf{pn8-gR6KV=?bI`EeKFt+3f>PKo5xxmSk$5%Ue*TjZ&0|H>jaNF zuC2Do(TQ1)ifgf(A>P)7K@pAkbJZ$OY?OC_;{mFMcrZhc7ty<$rx_SO>ej_M6=Mkc zoCNY#)vP|oX5Z%8ub}U#lXu#Vwg%1aAy_|6eHQQjB9qfpp>=xTvz^8FhF4B)-5w89 zBSKv~snQvb&^;`Y2W&p~6)`bNyc~NLwMOh<2p4p<*Dp=Wb?nva z-7;eGoAEFIbaysj=fx4P$ zHmaoP#_~&nc{?ZjaB%G9dI&m%^M8`DcqpOKdn0T3=HSqZ4Brocvmf+{C&-_Ir;%x6 zo&9N^Cg*Kmlx$)@8?e%LQ}JLvWA zlUXWCL*LvIjA>xTQZZr1d>ML-HmQo4Q6&cSSC8I8SEQs9q$N9yAupqcvP&KZX`eP32 ziUB7ur?FRcvvF|-YDrcZ%n2>xgNqk%b2U#rhvUQHWaFEx{(Q?+ttk&%w;Bx7zH!yl zpulx6ef7oxC`M2boL0}+ba3>Gl?NsT*Lrq=}QXrAry)1==LH@)E78O0HD@`!VRFEPCyr@y+gHdsV6WU^7wGK5t zEt#E(3+iDPGSORIxzDep!3IB5;U3TOrQVGuh;wesbUQG?cI`9JD2ru_ZuYmbPM?Fhudg<3J^93$gRH?Cdu zub7lxai~wdf)(>700bHF#CWyeJtiH524N|ha8Hq1P`G>GliDZI%>Yln+URU>KCFC; zlAeykOKVQg5JH%KOKL=C0N4t)V#7!Sg$CxBL?-cQQX{_=tE*{szfpI%^VIo9kvWRM zG#aSaTaz8~kEcdPTqP;kmYxwaL6h!*#NLLso^q2EW_O`(q(Xl{xD{xRoS`O+_g8A^4Gp;d*sd6tYw?nQz*#_*wvBY_E(8CS~VWFkUwMCSlw z+^VMjm*Yp9%Wnw3f4hDJXZ2bkPa7K96g;dEZNnI_;W#$EfuGzv!xZYYqM3^Ujw0Q3 zt)Yop*tYsTWyR9-;0A@ospf({Hl!gBJ9d@dBtIMZE7gs~98(_+_yKo_uArxq)A6p4 zpipgaF!otQ6KNM;{)Q{)voy%iC<}<^mF~l}UG~I_qc>j&u8i81J)&P&UYaXX53Svw zp@#vdF|89SI`N=)zF_Xp1qJteTN_oIK1amzy1tgx zo%&w=xwJaF_B2kNEuauL*(<*YYKKaj60rwQ7leVu>y@p7;SG$OeI0^M9Dh4fYWgij zBo~B$6{%>fRi*2Ea8p&Wp${+Zv^O0mdE=>7!=^EYPIIfi)DqS_=I{!OZ-lX8!Y+a7 zxLbP%rW8yqlAh0bf#G(3zDbq_VTiTc)cY7a-I#BoWq@ZeKkQ7L&DTxed(}>B^cWPf zgfE~H^=h4rVXo*H&%*7k+(3o!Zure^;u~GqFky7SV#qbotoA~ZS=uhBDp~F2QgL4F zlHv(I{JG_hS1j5M%&;%gvhN2rlz*S5tV}GV959;I!gvVV3SuertVi?DibUSUlAK4G zMqV!nQ^2JvLI-CT094T-A1-LbBK?J&dUvU5A)_nwPR%GZs6XC zn^}gsqapF}1f|e`b7dvj?~&)&!AN`~k}=Eg;72lo_E>GFLmD$={%We~a)}1WjN3rw zZJ%Y@#Ob~NA)MQ1#6@?Rtc1&d5>)?agqx!IYgWIo)kARUHEJ!0cmyO`+p?7}SGYz1 z7*LLbeoWSqyJa{Dc~Feo6l%Qx^I9FiS!rzd;+1IE-zvP?3q53Oq1{>)zEb5gRVTTF zBUR)kXvYZTmN3zx>k=Yal9JM-{G;8CP8}1~r0?-u=lpHy(`U2k`s-coQG; z**Do^&Nl5c5gFQU@@`$L3*Uxn8*U-{cXe@LF1AOGY&J6Nf7JY9_t3x%-!~|Z;Na{Q zeD6NgD@C~e#@v_T$JPh2A|eKLqDlSL8dj$?0{_{lQ?!LfmPgO|a|K^dfj0A<9+saf z9%Y%RX{W3JtzV{8xHJ$L{kM$3iJSlM41naVu0|^=+{G@Mc9h6l6M#K_am{jcKq zVv^q-F%jh6hWohk^nX&byWivx3#z*CF-6Cs)tKA}WeC?8&KL%wugnM>k_Is0j z8gzJn2D8dx&^Pgif2u61I|^n?=?MV?EnMesodWmOZKu1+Ur3+*`o?v-oU*Acf>a(_ z1xJs9StHuIl0VFQ|EVjQ+@rZ>A)=x}KoA?u)?=m^1?YZ6aGip@>!w3&hC-cMQfg&XJwbkH# zleUn52}RWG?W5}Y(lPP~OR74zvFvKET{!98q*F^>tkpS08AXHIPDlLIQAV5m?uUIp z&uUtufRnWGdut$i!~|9wDmJ1A&}LcMJcSVqMN(KV6&dqNPvB&{Bu5k%)sdQ z);T<{7M{D?R=%OtqFFMja-1wr%j)G#xa@rs^OG0bjod>Y^jD~B5O%PyL0B{dscCKg zR=NK5f4YtK=?ZaPDZgpr(@l>f(5J>pGa<`+&Ja0%H>!Y*)xk}*zZc zYgljb*5IaTz%|eOM8t)QI#h`JJkLb2z(9_H5K!giS%7zt3O${IHTOf(1Kpqr0*Ql4?d!+p17tny}4E!PV41EQ$&NX|r%U%N}um8;# z_F^}YNa&C37iq);s2l=NdC91n8FC(JmPg3^VkY0NJOdbu_r6}FVd(w@Mb46*p3ypN zx=I(^P$5Y@#2CQNE*|@=JT8mPV!KV62wxC3KfQs~8kH-lIU8IgN9|uHE0V1ZnDW2fij9C$e5-xCdi#f%N2BeN1BKm=Q60qK4%G;Mp z=?XKeU%8hrETLFxMN!zFU-yok>4(62U9jG&B-s-Y_^DF+rwgPb*t3|!mqqRXVpzT& zs=#-q7E8f1RDQGv74K=vLsL70Zl?pOzXGX?j4azX^=14Z@nlf!p4N9;uf4q5!6!V` zD8GE67mIJ+Z4fwwHe9R`3iuOU^PE(eoIbo_qr+MzNWL7pAUv=Sn8A3<44=KN56YPU z3U{kfzP8w0+07xmNOgNvky=W{fs86{cQ~XCe2USQ+=#gb-raje&d=jJSEhjeje-#h z+DfT)1h2<`dOBTy->h2=3B)^FwCy3x91;g>Ki|DwSw;6!2^?nA@WCi_7*&O)6gGie41(bIgpACGy_2VGeKtx=5>O)*t)G!A9_qD zf?HsjXkK+S`QB<9Db*oT%h?Cy6>p81r5hgyq70k+_x97>%AYQ9tY7r!ngykW`Qt5~ zuc3cv{p2wwFV>;?vK;o=uGPGQ75qs4tnarO4I|k0yC!@LMv<5(NnmFgu(Kj#=Zc(p zt>Ao#v=kYRv(_n)^sDTNh`ShlBXiqV55kx2-w`WCk6+iJ0}M$A{Z*CZPf4|HT97z$ z3Sc7X<)a4`h)ah~ntFecMqeBI?fINw?tq@=_<8CMd{8RbaWoigN$&@>Ihai7umbN{*T_Btv$0rh?$_L3V;(I_UcC8OH;8dFJnSSGxj&VZ zopdeXhQ2we!AAx|dOw=A#Z@MGicMAQX1x7W_+yTAua{HH>?>TNsk~wLht4QW>m#KO zs2*7K9O$=JCKn`8-vABiECng>eJl9D7nyk1VAFHW#wF|H+f|&Dx}*AKl;#C`YX$Ze3$1o$L zjLLI#-@7y9`4SBEnh&%Nh7OV7b;Nli5G$cBABapL47W)1csK4Y(8qf|M0@kt;MLSm zv&o&*H$9^WI{eXz5L{H@`mQo#uoCnW<{42IDwij*ZqM0$1KC-#1jhwm043aYkwZG1 z7$x}!#nj+l-=$6JgC7%G+O|(P`QB-rW^4-%gY%cS7r%u?0hb>_Dx{9f)p~ksTwV{h zC$CN0)9Ui<^u#(+*<06#h@}~;wSGvwT436JyS#71woIE_qjZV#(*)63TUX0z8c9(1 z^nSe-vh0h*mDL9mwE`!Cn8B;4%nt!JmtSGT3rp^V0}OSkuHsMSmsWXwT!ml;is%?s z;QS|jkB&-jzex3F!`I#x&k**sKWMX8kPBG-fwB7P&gzmFk6ei~0y-d7{l-68ZkJr} z2yfM9(k~T0f{Lh^QIK)qYJoId8tqHq2f?Zdj3fe_-&NpJwRSH6F;DJVT7y`nGRmjg zPbh^4d$`y5<9NWIGg?;jA1D8et~J#EtkfB#Z0#!pqP(1eWNV=9r9)(b9`>?*V1dcS zUZyn!gKYIgUT-aHjUwWYp^=7;fL*VdM(;I_iT2Zh+*_tB2BIzm8c zmOyE9TS^O4J9X1H>s4kuE;UgeVXipcPOXlQV^Q&>mD1AII&37ZDce_qEn2JxZ67 zyI~fSsh~TpZ-&U_W|!`)rOEPy^??{$29ADV%h8W0>mvXZ5L(SRgA=dKO}T&x(KSP| z)mb%ocke=76h5g{>PNrgneVl$=ztSo5CB)ecjhd*1vowll>!JM1aG=Qwzb|H1w-$Dc4IB=5?(T*|jT#0Ms@zT9iclQ7ctUW1ptO`?z zE=LiE{QKA5namx>zPN>kq58_T&%Rq$=K<&c2jC^!frDF2-~nj(X^Sj3)aDT{5N#7| zB1k3fBY zvsco89+#+TE2s-GmpX=2-r$VJ`Xs;yylwlg&AHVIzK$29<#HRpcN+osqDU7~S4+}o zwW9O;Gp8sH{ehV`ftxu!Pw_JL{eQgkB-1#~dq6)u`_%7W4WMpejJO|?aU z8&>=^oX~@FoMobpF+VMX#n$^d4kA-yF(?H;CkJe@O&qo6a?l!HmwUbN@<){CM^~52 z+D-&N`>_6LlNI&iX3{NSePU4>7>-rz?pa{`#?q8gA zUR&QgUnu*hdtboo+WS01AS1@bp*IH}f4w+}z@up=YF`k^zpcb<;Na_!^K4Jys11iN zU2o@KD3J3}^6^wS2L7f1v5&L=7UYtr`m+LgUdTzw_ZDFBHBaEtXE< ztxqLwchjMiWu zJ&s3Yu}gHX9oJw91l7@OBqP|Ek{%alPb|3|dHSq`ZX_9IyNp9{q#$R+KhYXu1>c*I zTJW~GHR49)Jn44u`-cd^;!?=S?AKTFexX|3QI*#Ynj}p#llkbbVI1~lKn!q&tyqcS zwUJjw&4SAlQ=MhQ&69Ctsn&#&7ZI>H=6A6knX)`>hjKDLTD#A{Q6IGuJT|mT345>c zWH8sS@Eb$3=GAQI<+wWZwRwhPT?nyVj3}J%l5`N$>O;(a`@Q7D^Mw%mdUUErJs^F# zYECO!xZfWHj1_S7cSbVc%_71}#yO%Gg67^!1Xe^n4emV5TG8<(P*#As&8wGvK6q2> zlG^HwpZil5R%4(EYnjQ>UO&Hko>?G7a(^09c$@Zhd#wU>J10I~d0|hnsKM=@NpM;G zGPRVpr|K&+BSmLzrLTTR$i7L|jn39RxR-Cnqu!%VQ&W0HeN2$M_!c-)?`9xD_=`AG zb&0?~G`_4-L#e)OvZ<&8`n*-X!wBXF8IPuNG%AaVp9Z7C3qP*SyMe|pXY+m4Z(O>o$o%f141E#f14dBlO=)X-;hje!L z*S^3D9`VuEnqRsZc*n0jfKKvMQCidII*vzsPUItAEA&r&i{v983~t*@QJ15<@*tvD|Z`&Z?k)rG&4h;RG##O1|q^$agCq87K00L$(H z#g;MpkkK=^=OPu1inDh5XjNO~#-=}9AjOhLS8fI7n~wVNk8%3MTifg?g0F;|LD@9( zL8s62Y#1h2aQuk$1KkJ3XG%M4&wZ4rDK2>0AiGKNdY2Pqo9??3f>$!`&bb&?4*NDN z)Q|uC)tgu!jpfZ?9s$?Tznx>$Zx?eXO1uKH7)4Ru^ZA44mkk0cu9ehpdIj+kREJjX z)Mm||Lt25`vZ6szc&HNeYOYMWjM7R zYQ@Ebg!U?Jm!wY@G56|hMd;hA3AH_PhUJXIYOOU+FS`gL7T;L(K@U5v8s!eVrsFG1mbBe=f!me z4Hk2CaPbG+?}Q*->yiq&$`etV@tS-?qMlhRrN&EYX z`e_4yF2-?RT$vFC=T5vG!JO9Pwq}{q_Bm+c{G*qIx%pwWT55c6#hTXAbrfcFwZ5IX zH~{R!2@Z)f91P!yJeC{Iyb+IRXB-%)T2`lgBd)C zX;Tr@NNP)=WiZUkjxPnFGy(kT#FnPt?+xKj#O8g9gCpZ?VS5{V_P2a-8~gePyW&cU z9+ml?=6qy8iyXwMB9*#$S19BTS=BiO5V!;g@bRX4B0iF#&_BJ8sk~YHu22W*rIUBv z*m}^~dz^5$I-j-uB9WwPJMd{y%4w&U{M@~W6$^fcaCV9Id4Pr!prOPloIoE_yDJFN zA%Qm6*UdMN2rF)xvqtIG^SOnRqeV^24GQ_-kCIGU{P~>aLi1HswZZTYdkmy9p^{#xNJ?k0g_R`Gd8Pu#w}c;;~{ZNhH*ZJomb=f)R(y^Fj>$+Iov8=f5v{iN?W_}tze=At)UH}s#d$jTw%fX6p&|?nGP}N2 z%JjuCB(N}nP6SD)yQSCqB2&G2S^o5i&Rm06Dk=WDN{7O#Y~PfMN2PZsQCdUK5b0r>mdKI;Vst09`)^wEzGB literal 0 HcmV?d00001 From 1291d962b2af71e56855c629432004635aaae4b6 Mon Sep 17 00:00:00 2001 From: ianbillett <33524269+ianbillett@users.noreply.github.com> Date: Tue, 12 Nov 2019 11:19:58 +0000 Subject: [PATCH 045/257] Docs: Expand ruler recommended alerting docs (#1719) * updates ruler docs Signed-off-by: Ian Billett * Update docs/components/rule.md Co-Authored-By: Bartlomiej Plotka Signed-off-by: Ian Billett * removes newline Signed-off-by: Ian Billett --- docs/components/rule.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/components/rule.md b/docs/components/rule.md index 55bac66749..359c0572a4 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -81,7 +81,7 @@ The most important metrics to alert on are: indicate connection, incompatibility or misconfiguration problems. * `prometheus_rule_evaluation_failures_total`. If greater than 0, it means that that rule failed to be evaluated, which results in -either gap in rule or potentially ignored alert. Alert heavily on this if this happens for longer than your alert thresholds. +either gap in rule or potentially ignored alert. This metric might indicate problems on the queryAPI endpoint you use. Alert heavily on this if this happens for longer than your alert thresholds. `strategy` label will tell you if failures comes from rules that tolerate [partial response](rule.md#partial-response) or not. * `prometheus_rule_group_last_duration_seconds < prometheus_rule_group_interval_seconds` If the difference is large, it means From a0747efd905ac9729018c11088fe1a640c76ff2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 13 Nov 2019 13:34:57 +0200 Subject: [PATCH 046/257] compact: add metric thanos_compactor_iterations_total (#1733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * compact: add metric thanos_compactor_iterations_total Add a metric called thanos_compactor_iterations_total that is a counter and will get increased by 1 every time an iteration gets executed successfully. This is needed in case --wait is specified and then our Compactor could die. We need to alert on such a case. One thing would be to alert on a restart of the container however that is not the most flexible thing - it might still be OK as long as it successfully finishes its job in time. However, it is impossible to know that exact part ATM. Add this metric so that users could add alerts like: ``` rate(thanos_compactor_iterations_total[1d]) == 0 FOR 3d ``` Signed-off-by: Giedrius Statkevičius * CHANGELOG: add entry Signed-off-by: Giedrius Statkevičius * compact: simplify wait check Signed-off-by: Giedrius Statkevičius * cmd: thanos: compact: remove wait check Let's register the metric no matter what since if it is run as a batch job then this metric does not matter either way. Signed-off-by: Giedrius Statkevičius * CHANGELOG: add period Add a period at the end of an item in the CHANGELOG to keep it uniform. Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 1 + cmd/thanos/compact.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8945f3425a..7f5463f2c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss) for further information. - [#1680](https://github.com/thanos-io/thanos/pull/1680) Add a new `--http-grace-period` CLI option to components which serve HTTP to set how long to wait until HTTP Server shuts down. - [#1712](https://github.com/thanos-io/thanos/pull/1712) Rename flag on bucket web component from `--listen` to `--http-address` to match other components. +- [#1733](https://github.com/thanos-io/thanos/pull/1733) New metric `thanos_compactor_iterations_total` on Thanos Compactor which shows the number of successful iterations. ### Fixed diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index ba09994913..2589a251ee 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -168,10 +168,15 @@ func runCompact( Name: "thanos_compactor_retries_total", Help: "Total number of retries after retriable compactor error", }) + iterations := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "thanos_compactor_iterations_total", + Help: "Total number of iterations that were executed successfully", + }) halted.Set(0) reg.MustRegister(halted) reg.MustRegister(retried) + reg.MustRegister(iterations) downsampleMetrics := newDownsampleMetrics(reg) @@ -313,6 +318,7 @@ func runCompact( return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { err := f() if err == nil { + iterations.Inc() return nil } From 5eb43c1280b7205bfcb008c2831851603cb420e6 Mon Sep 17 00:00:00 2001 From: Philip Panyukov Date: Wed, 13 Nov 2019 12:20:36 +0000 Subject: [PATCH 047/257] Do not generate XXX fields in protobufs (#1725) * Do not generate XXX fields in protobufs We don't need them. By gettig rid of them we achieve: - smaller memory footprint - smaller payloads on the wire - same in-memory layout with core Prometheus structs opening path for all sorts of optimisations such as unsafe casts, leading to better memory use and faster :) - see related issue: https://github.com/prometheus/prometheus/issues/6029 The tests show SG memory allocations dropped 5% for query execution: - Showing nodes accounting for -359.20MB, 6.19% of 5798.93MB total Signed-off-by: Philip Panyukov * comment to explain why do don't generate XXX fields in protobufs Signed-off-by: Philip Panyukov --- pkg/store/storepb/rpc.pb.go | 214 +++++++++------------------------- pkg/store/storepb/rpc.proto | 6 + pkg/store/storepb/types.pb.go | 146 ++++++++--------------- pkg/store/storepb/types.proto | 6 + 4 files changed, 116 insertions(+), 256 deletions(-) diff --git a/pkg/store/storepb/rpc.pb.go b/pkg/store/storepb/rpc.pb.go index 3c46313e66..a4a34bd29a 100644 --- a/pkg/store/storepb/rpc.pb.go +++ b/pkg/store/storepb/rpc.pb.go @@ -137,9 +137,6 @@ func (Aggr) EnumDescriptor() ([]byte, []int) { } type InfoRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } func (m *InfoRequest) Reset() { *m = InfoRequest{} } @@ -182,10 +179,7 @@ type InfoResponse struct { MaxTime int64 `protobuf:"varint,3,opt,name=max_time,json=maxTime,proto3" json:"max_time,omitempty"` StoreType StoreType `protobuf:"varint,4,opt,name=storeType,proto3,enum=thanos.StoreType" json:"storeType,omitempty"` // label_sets is an unsorted list of `LabelSet`s. - LabelSets []LabelSet `protobuf:"bytes,5,rep,name=label_sets,json=labelSets,proto3" json:"label_sets"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + LabelSets []LabelSet `protobuf:"bytes,5,rep,name=label_sets,json=labelSets,proto3" json:"label_sets"` } func (m *InfoResponse) Reset() { *m = InfoResponse{} } @@ -222,10 +216,7 @@ func (m *InfoResponse) XXX_DiscardUnknown() { var xxx_messageInfo_InfoResponse proto.InternalMessageInfo type LabelSet struct { - Labels []Label `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Labels []Label `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"` } func (m *LabelSet) Reset() { *m = LabelSet{} } @@ -271,9 +262,6 @@ type SeriesRequest struct { PartialResponseDisabled bool `protobuf:"varint,6,opt,name=partial_response_disabled,json=partialResponseDisabled,proto3" json:"partial_response_disabled,omitempty"` // TODO(bwplotka): Move Thanos components to use strategy instead. Including QueryAPI. PartialResponseStrategy PartialResponseStrategy `protobuf:"varint,7,opt,name=partial_response_strategy,json=partialResponseStrategy,proto3,enum=thanos.PartialResponseStrategy" json:"partial_response_strategy,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } func (m *SeriesRequest) Reset() { *m = SeriesRequest{} } @@ -313,10 +301,7 @@ type SeriesResponse struct { // Types that are valid to be assigned to Result: // *SeriesResponse_Series // *SeriesResponse_Warning - Result isSeriesResponse_Result `protobuf_oneof:"result"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Result isSeriesResponse_Result `protobuf_oneof:"result"` } func (m *SeriesResponse) Reset() { *m = SeriesResponse{} } @@ -463,9 +448,6 @@ type LabelNamesRequest struct { PartialResponseDisabled bool `protobuf:"varint,1,opt,name=partial_response_disabled,json=partialResponseDisabled,proto3" json:"partial_response_disabled,omitempty"` // TODO(bwplotka): Move Thanos components to use strategy instead. Including QueryAPI. PartialResponseStrategy PartialResponseStrategy `protobuf:"varint,2,opt,name=partial_response_strategy,json=partialResponseStrategy,proto3,enum=thanos.PartialResponseStrategy" json:"partial_response_strategy,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } func (m *LabelNamesRequest) Reset() { *m = LabelNamesRequest{} } @@ -502,11 +484,8 @@ func (m *LabelNamesRequest) XXX_DiscardUnknown() { var xxx_messageInfo_LabelNamesRequest proto.InternalMessageInfo type LabelNamesResponse struct { - Names []string `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"` - Warnings []string `protobuf:"bytes,2,rep,name=warnings,proto3" json:"warnings,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Names []string `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"` + Warnings []string `protobuf:"bytes,2,rep,name=warnings,proto3" json:"warnings,omitempty"` } func (m *LabelNamesResponse) Reset() { *m = LabelNamesResponse{} } @@ -547,9 +526,6 @@ type LabelValuesRequest struct { PartialResponseDisabled bool `protobuf:"varint,2,opt,name=partial_response_disabled,json=partialResponseDisabled,proto3" json:"partial_response_disabled,omitempty"` // TODO(bwplotka): Move Thanos components to use strategy instead. Including QueryAPI. PartialResponseStrategy PartialResponseStrategy `protobuf:"varint,3,opt,name=partial_response_strategy,json=partialResponseStrategy,proto3,enum=thanos.PartialResponseStrategy" json:"partial_response_strategy,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } func (m *LabelValuesRequest) Reset() { *m = LabelValuesRequest{} } @@ -586,11 +562,8 @@ func (m *LabelValuesRequest) XXX_DiscardUnknown() { var xxx_messageInfo_LabelValuesRequest proto.InternalMessageInfo type LabelValuesResponse struct { - Values []string `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"` - Warnings []string `protobuf:"bytes,2,rep,name=warnings,proto3" json:"warnings,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Values []string `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"` + Warnings []string `protobuf:"bytes,2,rep,name=warnings,proto3" json:"warnings,omitempty"` } func (m *LabelValuesResponse) Reset() { *m = LabelValuesResponse{} } @@ -644,56 +617,57 @@ func init() { func init() { proto.RegisterFile("rpc.proto", fileDescriptor_77a6da22d6a3feb1) } var fileDescriptor_77a6da22d6a3feb1 = []byte{ - // 772 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x4f, 0x6f, 0xfa, 0x46, - 0x10, 0x65, 0x6d, 0x30, 0x78, 0x48, 0x90, 0xb3, 0x21, 0x89, 0x71, 0x25, 0x82, 0x38, 0xa1, 0xb4, - 0x22, 0x2d, 0x55, 0x5b, 0xb5, 0x37, 0x20, 0x8e, 0x82, 0x9a, 0x40, 0xbb, 0x40, 0xe8, 0x9f, 0x03, - 0x35, 0xc9, 0xd6, 0xb1, 0x64, 0x6c, 0xea, 0x35, 0x4d, 0x72, 0xed, 0xe7, 0xe9, 0xb7, 0xe8, 0x25, - 0xc7, 0x5e, 0x7b, 0xa9, 0xda, 0x7c, 0x8a, 0x1e, 0x2b, 0xaf, 0xd7, 0x80, 0xdb, 0x24, 0xd2, 0x4f, - 0xdc, 0x76, 0xde, 0x1b, 0xcf, 0xec, 0x7b, 0x3b, 0xbb, 0x06, 0x35, 0x58, 0xdc, 0x34, 0x17, 0x81, - 0x1f, 0xfa, 0x58, 0x09, 0xef, 0x2c, 0xcf, 0x67, 0x46, 0x31, 0x7c, 0x5c, 0x50, 0x16, 0x83, 0x46, - 0xd9, 0xf6, 0x6d, 0x9f, 0x2f, 0x4f, 0xa3, 0x55, 0x8c, 0xd6, 0x77, 0xa1, 0xd8, 0xf3, 0x7e, 0xf4, - 0x09, 0xfd, 0x69, 0x49, 0x59, 0x58, 0xff, 0x03, 0xc1, 0x4e, 0x1c, 0xb3, 0x85, 0xef, 0x31, 0x8a, - 0xdf, 0x07, 0xc5, 0xb5, 0x66, 0xd4, 0x65, 0x3a, 0xaa, 0xc9, 0x8d, 0x62, 0x6b, 0xb7, 0x19, 0xd7, - 0x6e, 0x5e, 0x46, 0x68, 0x27, 0xfb, 0xf4, 0xe7, 0x71, 0x86, 0x88, 0x14, 0x5c, 0x81, 0xc2, 0xdc, - 0xf1, 0xa6, 0xa1, 0x33, 0xa7, 0xba, 0x54, 0x43, 0x0d, 0x99, 0xe4, 0xe7, 0x8e, 0x37, 0x72, 0xe6, - 0x94, 0x53, 0xd6, 0x43, 0x4c, 0xc9, 0x82, 0xb2, 0x1e, 0x38, 0x75, 0x0a, 0x2a, 0x0b, 0xfd, 0x80, - 0x8e, 0x1e, 0x17, 0x54, 0xcf, 0xd6, 0x50, 0xa3, 0xd4, 0xda, 0x4b, 0xba, 0x0c, 0x13, 0x82, 0xac, - 0x73, 0xf0, 0x27, 0x00, 0xbc, 0xe1, 0x94, 0xd1, 0x90, 0xe9, 0x39, 0xbe, 0x2f, 0x2d, 0xb5, 0xaf, - 0x21, 0x0d, 0xc5, 0xd6, 0x54, 0x57, 0xc4, 0xac, 0xfe, 0x19, 0x14, 0x12, 0xf2, 0x9d, 0x64, 0xd5, - 0xff, 0x91, 0x60, 0x77, 0x48, 0x03, 0x87, 0x32, 0x61, 0x53, 0x4a, 0x28, 0x7a, 0x5d, 0xa8, 0x94, - 0x16, 0xfa, 0x69, 0x44, 0x85, 0x37, 0x77, 0x34, 0x60, 0xba, 0xcc, 0xdb, 0x96, 0x53, 0x6d, 0xaf, - 0x62, 0x52, 0x74, 0x5f, 0xe5, 0xe2, 0x16, 0x1c, 0x44, 0x25, 0x03, 0xca, 0x7c, 0x77, 0x19, 0x3a, - 0xbe, 0x37, 0xbd, 0x77, 0xbc, 0x5b, 0xff, 0x9e, 0x9b, 0x25, 0x93, 0xfd, 0xb9, 0xf5, 0x40, 0x56, - 0xdc, 0x84, 0x53, 0xf8, 0x03, 0x00, 0xcb, 0xb6, 0x03, 0x6a, 0x5b, 0x21, 0x8d, 0x3d, 0x2a, 0xb5, - 0x76, 0x92, 0x6e, 0x6d, 0xdb, 0x0e, 0xc8, 0x06, 0x8f, 0xbf, 0x80, 0xca, 0xc2, 0x0a, 0x42, 0xc7, - 0x72, 0xa3, 0x2e, 0xfc, 0xe4, 0xa7, 0xb7, 0x0e, 0xb3, 0x66, 0x2e, 0xbd, 0xd5, 0x95, 0x1a, 0x6a, - 0x14, 0xc8, 0x91, 0x48, 0x48, 0x26, 0xe3, 0x4c, 0xd0, 0xf8, 0xfb, 0x17, 0xbe, 0x65, 0x61, 0x60, - 0x85, 0xd4, 0x7e, 0xd4, 0xf3, 0xfc, 0x38, 0x8f, 0x93, 0xc6, 0x5f, 0xa5, 0x6b, 0x0c, 0x45, 0xda, - 0xff, 0x8a, 0x27, 0x44, 0xfd, 0x07, 0x28, 0x25, 0xce, 0x8b, 0x81, 0x6c, 0x80, 0xc2, 0x38, 0xc2, - 0x8d, 0x2f, 0xb6, 0x4a, 0xab, 0x51, 0xe1, 0xe8, 0x45, 0x86, 0x08, 0x1e, 0x1b, 0x90, 0xbf, 0xb7, - 0x02, 0xcf, 0xf1, 0x6c, 0x7e, 0x10, 0xea, 0x45, 0x86, 0x24, 0x40, 0xa7, 0x00, 0x4a, 0x40, 0xd9, - 0xd2, 0x0d, 0xeb, 0xbf, 0x22, 0xd8, 0xe3, 0xee, 0xf7, 0xad, 0xf9, 0xfa, 0x80, 0xdf, 0x34, 0x04, - 0x6d, 0x61, 0x88, 0xb4, 0xa5, 0x21, 0xe7, 0x80, 0x37, 0x77, 0x2b, 0x4c, 0x29, 0x43, 0xce, 0x8b, - 0x00, 0x3e, 0xcd, 0x2a, 0x89, 0x03, 0x6c, 0x40, 0x41, 0xe8, 0x65, 0xba, 0xc4, 0x89, 0x55, 0x5c, - 0xff, 0x0d, 0x89, 0x42, 0xd7, 0x96, 0xbb, 0x5c, 0xeb, 0x2e, 0x43, 0x8e, 0x0f, 0x3d, 0xd7, 0xa8, - 0x92, 0x38, 0x78, 0xdb, 0x0d, 0x69, 0x0b, 0x37, 0xe4, 0x2d, 0xdd, 0xe8, 0xc1, 0x7e, 0x4a, 0x84, - 0xb0, 0xe3, 0x10, 0x94, 0x9f, 0x39, 0x22, 0xfc, 0x10, 0xd1, 0x5b, 0x86, 0x9c, 0x10, 0x50, 0x57, - 0x8f, 0x0d, 0x2e, 0x42, 0x7e, 0xdc, 0xff, 0xb2, 0x3f, 0x98, 0xf4, 0xb5, 0x0c, 0x56, 0x21, 0xf7, - 0xf5, 0xd8, 0x24, 0xdf, 0x6a, 0x08, 0x17, 0x20, 0x4b, 0xc6, 0x97, 0xa6, 0x26, 0x45, 0x19, 0xc3, - 0xde, 0x99, 0xd9, 0x6d, 0x13, 0x4d, 0x8e, 0x32, 0x86, 0xa3, 0x01, 0x31, 0xb5, 0x6c, 0x84, 0x13, - 0xb3, 0x6b, 0xf6, 0xae, 0x4d, 0x2d, 0x77, 0xd2, 0x84, 0xa3, 0x57, 0x24, 0x45, 0x95, 0x26, 0x6d, - 0x22, 0xca, 0xb7, 0x3b, 0x03, 0x32, 0xd2, 0xd0, 0x49, 0x07, 0xb2, 0xd1, 0xd5, 0xc4, 0x79, 0x90, - 0x49, 0x7b, 0x12, 0x73, 0xdd, 0xc1, 0xb8, 0x3f, 0xd2, 0x50, 0x84, 0x0d, 0xc7, 0x57, 0x9a, 0x14, - 0x2d, 0xae, 0x7a, 0x7d, 0x4d, 0xe6, 0x8b, 0xf6, 0x37, 0x71, 0x4f, 0x9e, 0x65, 0x12, 0x2d, 0xd7, - 0xfa, 0x45, 0x82, 0x1c, 0x17, 0x82, 0x3f, 0x82, 0x6c, 0xf4, 0x94, 0xe3, 0xfd, 0xc4, 0xde, 0x8d, - 0x87, 0xde, 0x28, 0xa7, 0x41, 0x61, 0xdc, 0xe7, 0xa0, 0xc4, 0xd7, 0x08, 0x1f, 0xa4, 0xaf, 0x55, - 0xf2, 0xd9, 0xe1, 0x7f, 0xe1, 0xf8, 0xc3, 0x0f, 0x11, 0xee, 0x02, 0xac, 0x07, 0x13, 0x57, 0x52, - 0x0f, 0xdb, 0xe6, 0xd5, 0x32, 0x8c, 0x97, 0x28, 0xd1, 0xff, 0x1c, 0x8a, 0x1b, 0xe7, 0x89, 0xd3, - 0xa9, 0xa9, 0x49, 0x35, 0xde, 0x7b, 0x91, 0x8b, 0xeb, 0x74, 0x2a, 0x4f, 0x7f, 0x57, 0x33, 0x4f, - 0xcf, 0x55, 0xf4, 0xfb, 0x73, 0x15, 0xfd, 0xf5, 0x5c, 0x45, 0xdf, 0xe5, 0xf9, 0xef, 0x63, 0x31, - 0x9b, 0x29, 0xfc, 0xbf, 0xf7, 0xf1, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe8, 0xcb, 0x18, 0x01, - 0x2f, 0x07, 0x00, 0x00, + // 791 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xcd, 0x6e, 0xeb, 0x44, + 0x14, 0xf6, 0xd8, 0x89, 0x13, 0x9f, 0xdc, 0x46, 0xbe, 0xd3, 0xdc, 0x7b, 0x5d, 0x23, 0xe5, 0x46, + 0x91, 0x90, 0xa2, 0x82, 0x72, 0x21, 0x08, 0x10, 0xec, 0x92, 0xd4, 0x55, 0x23, 0xda, 0x04, 0x26, + 0x49, 0xc3, 0xcf, 0x22, 0x38, 0xed, 0xe0, 0x5a, 0x72, 0xec, 0xe0, 0x71, 0x68, 0xbb, 0xe5, 0x09, + 0x78, 0x10, 0xde, 0x82, 0x4d, 0x97, 0x5d, 0xc2, 0x06, 0x41, 0xfb, 0x12, 0x2c, 0x91, 0xc7, 0xe3, + 0x24, 0x86, 0xb6, 0xd2, 0x55, 0x76, 0x73, 0xbe, 0xef, 0xf8, 0x9c, 0xf9, 0xbe, 0x39, 0x33, 0x06, + 0x2d, 0x5c, 0x9c, 0x35, 0x17, 0x61, 0x10, 0x05, 0x58, 0x8d, 0x2e, 0x6c, 0x3f, 0x60, 0x66, 0x29, + 0xba, 0x5e, 0x50, 0x96, 0x80, 0x66, 0xc5, 0x09, 0x9c, 0x80, 0x2f, 0xdf, 0xc4, 0xab, 0x04, 0xad, + 0xef, 0x40, 0xa9, 0xe7, 0xff, 0x10, 0x10, 0xfa, 0xe3, 0x92, 0xb2, 0xa8, 0xfe, 0x07, 0x82, 0x67, + 0x49, 0xcc, 0x16, 0x81, 0xcf, 0x28, 0x7e, 0x0f, 0x54, 0xcf, 0x9e, 0x51, 0x8f, 0x19, 0xa8, 0xa6, + 0x34, 0x4a, 0xad, 0x9d, 0x66, 0x52, 0xbb, 0x79, 0x1c, 0xa3, 0x9d, 0xdc, 0xcd, 0x9f, 0xaf, 0x25, + 0x22, 0x52, 0xf0, 0x1e, 0x14, 0xe7, 0xae, 0x3f, 0x8d, 0xdc, 0x39, 0x35, 0xe4, 0x1a, 0x6a, 0x28, + 0xa4, 0x30, 0x77, 0xfd, 0x91, 0x3b, 0xa7, 0x9c, 0xb2, 0xaf, 0x12, 0x4a, 0x11, 0x94, 0x7d, 0xc5, + 0xa9, 0x37, 0xa0, 0xb1, 0x28, 0x08, 0xe9, 0xe8, 0x7a, 0x41, 0x8d, 0x5c, 0x0d, 0x35, 0xca, 0xad, + 0xe7, 0x69, 0x97, 0x61, 0x4a, 0x90, 0x75, 0x0e, 0xfe, 0x18, 0x80, 0x37, 0x9c, 0x32, 0x1a, 0x31, + 0x23, 0xcf, 0xf7, 0xa5, 0x67, 0xf6, 0x35, 0xa4, 0x91, 0xd8, 0x9a, 0xe6, 0x89, 0x98, 0xd5, 0x3f, + 0x85, 0x62, 0x4a, 0xbe, 0x95, 0xac, 0xfa, 0x3f, 0x32, 0xec, 0x0c, 0x69, 0xe8, 0x52, 0x26, 0x6c, + 0xca, 0x08, 0x45, 0x8f, 0x0b, 0x95, 0xb3, 0x42, 0x3f, 0x89, 0xa9, 0xe8, 0xec, 0x82, 0x86, 0xcc, + 0x50, 0x78, 0xdb, 0x4a, 0xa6, 0xed, 0x49, 0x42, 0x8a, 0xee, 0xab, 0x5c, 0xdc, 0x82, 0x17, 0x71, + 0xc9, 0x90, 0xb2, 0xc0, 0x5b, 0x46, 0x6e, 0xe0, 0x4f, 0x2f, 0x5d, 0xff, 0x3c, 0xb8, 0xe4, 0x66, + 0x29, 0x64, 0x77, 0x6e, 0x5f, 0x91, 0x15, 0x37, 0xe1, 0x14, 0x7e, 0x1f, 0xc0, 0x76, 0x9c, 0x90, + 0x3a, 0x76, 0x44, 0x13, 0x8f, 0xca, 0xad, 0x67, 0x69, 0xb7, 0xb6, 0xe3, 0x84, 0x64, 0x83, 0xc7, + 0x9f, 0xc3, 0xde, 0xc2, 0x0e, 0x23, 0xd7, 0xf6, 0xe2, 0x2e, 0xfc, 0xe4, 0xa7, 0xe7, 0x2e, 0xb3, + 0x67, 0x1e, 0x3d, 0x37, 0xd4, 0x1a, 0x6a, 0x14, 0xc9, 0x2b, 0x91, 0x90, 0x4e, 0xc6, 0x81, 0xa0, + 0xf1, 0x77, 0x0f, 0x7c, 0xcb, 0xa2, 0xd0, 0x8e, 0xa8, 0x73, 0x6d, 0x14, 0xf8, 0x71, 0xbe, 0x4e, + 0x1b, 0x7f, 0x99, 0xad, 0x31, 0x14, 0x69, 0xff, 0x2b, 0x9e, 0x12, 0xf5, 0xef, 0xa1, 0x9c, 0x3a, + 0x2f, 0x06, 0xb2, 0x01, 0x2a, 0xe3, 0x08, 0x37, 0xbe, 0xd4, 0x2a, 0xaf, 0x46, 0x85, 0xa3, 0x47, + 0x12, 0x11, 0x3c, 0x36, 0xa1, 0x70, 0x69, 0x87, 0xbe, 0xeb, 0x3b, 0xfc, 0x20, 0xb4, 0x23, 0x89, + 0xa4, 0x40, 0xa7, 0x08, 0x6a, 0x48, 0xd9, 0xd2, 0x8b, 0xea, 0xbf, 0x22, 0x78, 0xce, 0xdd, 0xef, + 0xdb, 0xf3, 0xf5, 0x01, 0x3f, 0x69, 0x08, 0xda, 0xc2, 0x10, 0x79, 0x4b, 0x43, 0x0e, 0x01, 0x6f, + 0xee, 0x56, 0x98, 0x52, 0x81, 0xbc, 0x1f, 0x03, 0x7c, 0x9a, 0x35, 0x92, 0x04, 0xd8, 0x84, 0xa2, + 0xd0, 0xcb, 0x0c, 0x99, 0x13, 0xab, 0xb8, 0xfe, 0x1b, 0x12, 0x85, 0x4e, 0x6d, 0x6f, 0xb9, 0xd6, + 0x5d, 0x81, 0x3c, 0x1f, 0x7a, 0xae, 0x51, 0x23, 0x49, 0xf0, 0xb4, 0x1b, 0xf2, 0x16, 0x6e, 0x28, + 0x5b, 0xba, 0xd1, 0x83, 0xdd, 0x8c, 0x08, 0x61, 0xc7, 0x4b, 0x50, 0x7f, 0xe2, 0x88, 0xf0, 0x43, + 0x44, 0x4f, 0x19, 0xb2, 0x4f, 0x40, 0x5b, 0x3d, 0x36, 0xb8, 0x04, 0x85, 0x71, 0xff, 0x8b, 0xfe, + 0x60, 0xd2, 0xd7, 0x25, 0xac, 0x41, 0xfe, 0xab, 0xb1, 0x45, 0xbe, 0xd1, 0x11, 0x2e, 0x42, 0x8e, + 0x8c, 0x8f, 0x2d, 0x5d, 0x8e, 0x33, 0x86, 0xbd, 0x03, 0xab, 0xdb, 0x26, 0xba, 0x12, 0x67, 0x0c, + 0x47, 0x03, 0x62, 0xe9, 0xb9, 0x18, 0x27, 0x56, 0xd7, 0xea, 0x9d, 0x5a, 0x7a, 0x7e, 0xbf, 0x09, + 0xaf, 0x1e, 0x91, 0x14, 0x57, 0x9a, 0xb4, 0x89, 0x28, 0xdf, 0xee, 0x0c, 0xc8, 0x48, 0x47, 0xfb, + 0x1d, 0xc8, 0xc5, 0x57, 0x13, 0x17, 0x40, 0x21, 0xed, 0x49, 0xc2, 0x75, 0x07, 0xe3, 0xfe, 0x48, + 0x47, 0x31, 0x36, 0x1c, 0x9f, 0xe8, 0x72, 0xbc, 0x38, 0xe9, 0xf5, 0x75, 0x85, 0x2f, 0xda, 0x5f, + 0x27, 0x3d, 0x79, 0x96, 0x45, 0xf4, 0x7c, 0xeb, 0x67, 0x19, 0xf2, 0x5c, 0x08, 0xfe, 0x10, 0x72, + 0xf1, 0x53, 0x8e, 0x77, 0x53, 0x7b, 0x37, 0x1e, 0x7a, 0xb3, 0x92, 0x05, 0x85, 0x71, 0x9f, 0x81, + 0x9a, 0x5c, 0x23, 0xfc, 0x22, 0x7b, 0xad, 0xd2, 0xcf, 0x5e, 0xfe, 0x17, 0x4e, 0x3e, 0xfc, 0x00, + 0xe1, 0x2e, 0xc0, 0x7a, 0x30, 0xf1, 0x5e, 0xe6, 0x61, 0xdb, 0xbc, 0x5a, 0xa6, 0xf9, 0x10, 0x25, + 0xfa, 0x1f, 0x42, 0x69, 0xe3, 0x3c, 0x71, 0x36, 0x35, 0x33, 0xa9, 0xe6, 0x3b, 0x0f, 0x72, 0x49, + 0x9d, 0xce, 0xbb, 0x37, 0x7f, 0x57, 0xa5, 0x9b, 0xbb, 0x2a, 0xba, 0xbd, 0xab, 0xa2, 0xbf, 0xee, + 0xaa, 0xe8, 0x97, 0xfb, 0xaa, 0x74, 0x7b, 0x5f, 0x95, 0x7e, 0xbf, 0xaf, 0x4a, 0xdf, 0x16, 0xf8, + 0xaf, 0x64, 0x31, 0x9b, 0xa9, 0xfc, 0x1f, 0xf8, 0xd1, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x5c, + 0x24, 0xf3, 0x5b, 0x3b, 0x07, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -954,10 +928,6 @@ func (m *InfoRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } return len(dAtA) - i, nil } @@ -981,10 +951,6 @@ func (m *InfoResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.LabelSets) > 0 { for iNdEx := len(m.LabelSets) - 1; iNdEx >= 0; iNdEx-- { { @@ -1051,10 +1017,6 @@ func (m *LabelSet) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Labels) > 0 { for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { { @@ -1092,10 +1054,6 @@ func (m *SeriesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if m.PartialResponseStrategy != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.PartialResponseStrategy)) i-- @@ -1181,10 +1139,6 @@ func (m *SeriesResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if m.Result != nil { { size := m.Result.Size() @@ -1250,10 +1204,6 @@ func (m *LabelNamesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if m.PartialResponseStrategy != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.PartialResponseStrategy)) i-- @@ -1292,10 +1242,6 @@ func (m *LabelNamesResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Warnings) > 0 { for iNdEx := len(m.Warnings) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Warnings[iNdEx]) @@ -1337,10 +1283,6 @@ func (m *LabelValuesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if m.PartialResponseStrategy != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.PartialResponseStrategy)) i-- @@ -1386,10 +1328,6 @@ func (m *LabelValuesResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Warnings) > 0 { for iNdEx := len(m.Warnings) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Warnings[iNdEx]) @@ -1428,9 +1366,6 @@ func (m *InfoRequest) Size() (n int) { } var l int _ = l - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -1461,9 +1396,6 @@ func (m *InfoResponse) Size() (n int) { n += 1 + l + sovRpc(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -1479,9 +1411,6 @@ func (m *LabelSet) Size() (n int) { n += 1 + l + sovRpc(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -1519,9 +1448,6 @@ func (m *SeriesRequest) Size() (n int) { if m.PartialResponseStrategy != 0 { n += 1 + sovRpc(uint64(m.PartialResponseStrategy)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -1534,9 +1460,6 @@ func (m *SeriesResponse) Size() (n int) { if m.Result != nil { n += m.Result.Size() } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -1574,9 +1497,6 @@ func (m *LabelNamesRequest) Size() (n int) { if m.PartialResponseStrategy != 0 { n += 1 + sovRpc(uint64(m.PartialResponseStrategy)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -1598,9 +1518,6 @@ func (m *LabelNamesResponse) Size() (n int) { n += 1 + l + sovRpc(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -1620,9 +1537,6 @@ func (m *LabelValuesRequest) Size() (n int) { if m.PartialResponseStrategy != 0 { n += 1 + sovRpc(uint64(m.PartialResponseStrategy)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -1644,9 +1558,6 @@ func (m *LabelValuesResponse) Size() (n int) { n += 1 + l + sovRpc(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -1700,7 +1611,6 @@ func (m *InfoRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -1879,7 +1789,6 @@ func (m *InfoResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -1967,7 +1876,6 @@ func (m *LabelSet) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -2220,7 +2128,6 @@ func (m *SeriesRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -2341,7 +2248,6 @@ func (m *SeriesResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -2434,7 +2340,6 @@ func (m *LabelNamesRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -2552,7 +2457,6 @@ func (m *LabelNamesResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -2677,7 +2581,6 @@ func (m *LabelValuesRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -2795,7 +2698,6 @@ func (m *LabelValuesResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } diff --git a/pkg/store/storepb/rpc.proto b/pkg/store/storepb/rpc.proto index 8b2098e43c..278d41f350 100644 --- a/pkg/store/storepb/rpc.proto +++ b/pkg/store/storepb/rpc.proto @@ -11,6 +11,12 @@ option (gogoproto.marshaler_all) = true; option (gogoproto.unmarshaler_all) = true; option (gogoproto.goproto_getters_all) = false; +// Do not generate XXX fields to reduce memory footprint and opening a door +// for zero-copy casts to/from prometheus data types. +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.goproto_sizecache_all) = false; + /// Store reprents API against instance that stores XOR encoded values with label set metadata (e.g Prometheus metrics). service Store { /// Info returns meta information about a store e.g labels that makes that store unique as well as time range that is diff --git a/pkg/store/storepb/types.pb.go b/pkg/store/storepb/types.pb.go index 86a8b06237..5a8ae93fce 100644 --- a/pkg/store/storepb/types.pb.go +++ b/pkg/store/storepb/types.pb.go @@ -78,11 +78,8 @@ func (LabelMatcher_Type) EnumDescriptor() ([]byte, []int) { } type Label struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` } func (m *Label) Reset() { *m = Label{} } @@ -119,11 +116,8 @@ func (m *Label) XXX_DiscardUnknown() { var xxx_messageInfo_Label proto.InternalMessageInfo type Chunk struct { - Type Chunk_Encoding `protobuf:"varint,1,opt,name=type,proto3,enum=thanos.Chunk_Encoding" json:"type,omitempty"` - Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Type Chunk_Encoding `protobuf:"varint,1,opt,name=type,proto3,enum=thanos.Chunk_Encoding" json:"type,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` } func (m *Chunk) Reset() { *m = Chunk{} } @@ -160,11 +154,8 @@ func (m *Chunk) XXX_DiscardUnknown() { var xxx_messageInfo_Chunk proto.InternalMessageInfo type Series struct { - Labels []Label `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"` - Chunks []AggrChunk `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Labels []Label `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"` + Chunks []AggrChunk `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks"` } func (m *Series) Reset() { *m = Series{} } @@ -201,17 +192,14 @@ func (m *Series) XXX_DiscardUnknown() { var xxx_messageInfo_Series proto.InternalMessageInfo type AggrChunk struct { - MinTime int64 `protobuf:"varint,1,opt,name=min_time,json=minTime,proto3" json:"min_time,omitempty"` - MaxTime int64 `protobuf:"varint,2,opt,name=max_time,json=maxTime,proto3" json:"max_time,omitempty"` - Raw *Chunk `protobuf:"bytes,3,opt,name=raw,proto3" json:"raw,omitempty"` - Count *Chunk `protobuf:"bytes,4,opt,name=count,proto3" json:"count,omitempty"` - Sum *Chunk `protobuf:"bytes,5,opt,name=sum,proto3" json:"sum,omitempty"` - Min *Chunk `protobuf:"bytes,6,opt,name=min,proto3" json:"min,omitempty"` - Max *Chunk `protobuf:"bytes,7,opt,name=max,proto3" json:"max,omitempty"` - Counter *Chunk `protobuf:"bytes,8,opt,name=counter,proto3" json:"counter,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MinTime int64 `protobuf:"varint,1,opt,name=min_time,json=minTime,proto3" json:"min_time,omitempty"` + MaxTime int64 `protobuf:"varint,2,opt,name=max_time,json=maxTime,proto3" json:"max_time,omitempty"` + Raw *Chunk `protobuf:"bytes,3,opt,name=raw,proto3" json:"raw,omitempty"` + Count *Chunk `protobuf:"bytes,4,opt,name=count,proto3" json:"count,omitempty"` + Sum *Chunk `protobuf:"bytes,5,opt,name=sum,proto3" json:"sum,omitempty"` + Min *Chunk `protobuf:"bytes,6,opt,name=min,proto3" json:"min,omitempty"` + Max *Chunk `protobuf:"bytes,7,opt,name=max,proto3" json:"max,omitempty"` + Counter *Chunk `protobuf:"bytes,8,opt,name=counter,proto3" json:"counter,omitempty"` } func (m *AggrChunk) Reset() { *m = AggrChunk{} } @@ -249,12 +237,9 @@ var xxx_messageInfo_AggrChunk proto.InternalMessageInfo // Matcher specifies a rule, which can match or set of labels or not. type LabelMatcher struct { - Type LabelMatcher_Type `protobuf:"varint,1,opt,name=type,proto3,enum=thanos.LabelMatcher_Type" json:"type,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Value string `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Type LabelMatcher_Type `protobuf:"varint,1,opt,name=type,proto3,enum=thanos.LabelMatcher_Type" json:"type,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Value string `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` } func (m *LabelMatcher) Reset() { *m = LabelMatcher{} } @@ -303,34 +288,35 @@ func init() { func init() { proto.RegisterFile("types.proto", fileDescriptor_d938547f84707355) } var fileDescriptor_d938547f84707355 = []byte{ - // 432 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xdd, 0x6e, 0xd3, 0x30, - 0x14, 0xc7, 0xeb, 0x7c, 0x76, 0x67, 0x03, 0x05, 0x33, 0x21, 0x97, 0x8b, 0xae, 0x0a, 0x17, 0x54, - 0x20, 0x32, 0x31, 0x9e, 0x80, 0xa1, 0xdc, 0xf1, 0xa1, 0x99, 0x5d, 0x20, 0x84, 0x84, 0xdc, 0xce, - 0xa4, 0x11, 0x8d, 0x5d, 0xc5, 0x0e, 0x74, 0x8f, 0x81, 0x78, 0xa9, 0x5e, 0xf2, 0x04, 0x08, 0xfa, - 0x24, 0xc8, 0x27, 0x0d, 0x5b, 0xa5, 0xdc, 0x1d, 0x9f, 0xff, 0xef, 0x7c, 0xc8, 0xe7, 0x0f, 0x87, - 0xf6, 0x7a, 0x25, 0x4d, 0xb6, 0xaa, 0xb5, 0xd5, 0x34, 0xb2, 0x0b, 0xa1, 0xb4, 0x79, 0x78, 0x5c, - 0xe8, 0x42, 0x63, 0xea, 0xd4, 0x45, 0xad, 0x9a, 0x3e, 0x87, 0xf0, 0xb5, 0x98, 0xc9, 0x25, 0xa5, - 0x10, 0x28, 0x51, 0x49, 0x46, 0x26, 0x64, 0x7a, 0xc0, 0x31, 0xa6, 0xc7, 0x10, 0x7e, 0x13, 0xcb, - 0x46, 0x32, 0x0f, 0x93, 0xed, 0x23, 0xfd, 0x04, 0xe1, 0xab, 0x45, 0xa3, 0xbe, 0xd2, 0x27, 0x10, - 0xb8, 0x41, 0x58, 0x72, 0xf7, 0xec, 0x41, 0xd6, 0x0e, 0xca, 0x50, 0xcc, 0x72, 0x35, 0xd7, 0x57, - 0xa5, 0x2a, 0x38, 0x32, 0xae, 0xfd, 0x95, 0xb0, 0x02, 0x3b, 0x1d, 0x71, 0x8c, 0xd3, 0xfb, 0x30, - 0xec, 0x28, 0x1a, 0x83, 0xff, 0xe1, 0x1d, 0x4f, 0x06, 0xe9, 0x17, 0x88, 0xde, 0xcb, 0xba, 0x94, - 0x86, 0x3e, 0x85, 0x68, 0xe9, 0x56, 0x33, 0x8c, 0x4c, 0xfc, 0xe9, 0xe1, 0xd9, 0x9d, 0x6e, 0x00, - 0x2e, 0x7c, 0x1e, 0x6c, 0x7e, 0x9f, 0x0c, 0xf8, 0x0e, 0xa1, 0xa7, 0x10, 0xcd, 0xdd, 0x5c, 0xc3, - 0x3c, 0x84, 0xef, 0x75, 0xf0, 0xcb, 0xa2, 0xa8, 0x71, 0xa3, 0xae, 0xa0, 0xc5, 0xd2, 0x9f, 0x1e, - 0x1c, 0xfc, 0xd7, 0xe8, 0x08, 0x86, 0x55, 0xa9, 0x3e, 0xdb, 0x72, 0xf7, 0x03, 0x3e, 0x8f, 0xab, - 0x52, 0x5d, 0x96, 0x95, 0x44, 0x49, 0xac, 0x5b, 0xc9, 0xdb, 0x49, 0x62, 0x8d, 0xd2, 0x09, 0xf8, - 0xb5, 0xf8, 0xce, 0xfc, 0x09, 0xb9, 0xbd, 0x1e, 0x76, 0xe4, 0x4e, 0xa1, 0x8f, 0x20, 0x9c, 0xeb, - 0x46, 0x59, 0x16, 0xf4, 0x21, 0xad, 0xe6, 0xba, 0x98, 0xa6, 0x62, 0x61, 0x6f, 0x17, 0xd3, 0x54, - 0x0e, 0xa8, 0x4a, 0xc5, 0xa2, 0x5e, 0xa0, 0x2a, 0x15, 0x02, 0x62, 0xcd, 0xe2, 0x7e, 0x40, 0xac, - 0xe9, 0x63, 0x88, 0x71, 0x96, 0xac, 0xd9, 0xb0, 0x0f, 0xea, 0xd4, 0xf4, 0x07, 0x81, 0x23, 0xfc, - 0xde, 0x37, 0xc2, 0xce, 0x17, 0xb2, 0xa6, 0xcf, 0xf6, 0x6e, 0x3c, 0xda, 0x3b, 0xc1, 0x8e, 0xc9, - 0x2e, 0xaf, 0x57, 0xf2, 0xe6, 0xcc, 0xe8, 0x22, 0xaf, 0xcf, 0x45, 0xfe, 0x6d, 0x17, 0x4d, 0x21, - 0x70, 0x75, 0x34, 0x02, 0x2f, 0xbf, 0x48, 0x06, 0xce, 0x00, 0x6f, 0xf3, 0x8b, 0x84, 0xb8, 0x04, - 0xcf, 0x13, 0x0f, 0x13, 0x3c, 0x4f, 0xfc, 0xf3, 0xd1, 0xe6, 0xef, 0x78, 0xb0, 0xd9, 0x8e, 0xc9, - 0xaf, 0xed, 0x98, 0xfc, 0xd9, 0x8e, 0xc9, 0xc7, 0xd8, 0x58, 0x5d, 0xcb, 0xd5, 0x6c, 0x16, 0xa1, - 0x89, 0x5f, 0xfc, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x46, 0x5a, 0xe2, 0x68, 0xf1, 0x02, 0x00, 0x00, + // 445 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0xc7, 0xbd, 0xfe, 0x4c, 0xa7, 0x05, 0x99, 0xa5, 0x42, 0x5b, 0x0e, 0x6e, 0x64, 0x84, 0x88, + 0x40, 0xb8, 0xa2, 0x3c, 0x01, 0x45, 0xbe, 0xf1, 0xa1, 0x2e, 0x3d, 0x20, 0x84, 0x84, 0x36, 0xe9, + 0xe2, 0x58, 0xc4, 0xeb, 0xc8, 0x5e, 0x43, 0xfa, 0x16, 0x20, 0x5e, 0x2a, 0xc7, 0x1e, 0x39, 0x21, + 0x48, 0x5e, 0x04, 0xed, 0xd8, 0xa6, 0xad, 0xe4, 0xdb, 0xec, 0xfc, 0x7f, 0xf3, 0xa1, 0x9d, 0x3f, + 0xec, 0xea, 0x8b, 0xa5, 0xac, 0x93, 0x65, 0x55, 0xea, 0x92, 0xfa, 0x7a, 0x2e, 0x54, 0x59, 0xdf, + 0xdf, 0xcf, 0xca, 0xac, 0xc4, 0xd4, 0x91, 0x89, 0x5a, 0x35, 0x7e, 0x06, 0xde, 0x2b, 0x31, 0x95, + 0x0b, 0x4a, 0xc1, 0x55, 0xa2, 0x90, 0x8c, 0x8c, 0xc9, 0x64, 0x87, 0x63, 0x4c, 0xf7, 0xc1, 0xfb, + 0x2a, 0x16, 0x8d, 0x64, 0x36, 0x26, 0xdb, 0x47, 0xfc, 0x11, 0xbc, 0x97, 0xf3, 0x46, 0x7d, 0xa1, + 0x8f, 0xc1, 0x35, 0x83, 0xb0, 0xe4, 0xf6, 0xf1, 0xbd, 0xa4, 0x1d, 0x94, 0xa0, 0x98, 0xa4, 0x6a, + 0x56, 0x9e, 0xe7, 0x2a, 0xe3, 0xc8, 0x98, 0xf6, 0xe7, 0x42, 0x0b, 0xec, 0xb4, 0xc7, 0x31, 0x8e, + 0xef, 0xc2, 0xa8, 0xa7, 0x68, 0x00, 0xce, 0xfb, 0xb7, 0x3c, 0xb4, 0xe2, 0xcf, 0xe0, 0xbf, 0x93, + 0x55, 0x2e, 0x6b, 0xfa, 0x04, 0xfc, 0x85, 0x59, 0xad, 0x66, 0x64, 0xec, 0x4c, 0x76, 0x8f, 0x6f, + 0xf5, 0x03, 0x70, 0xe1, 0x13, 0x77, 0xfd, 0xfb, 0xd0, 0xe2, 0x1d, 0x42, 0x8f, 0xc0, 0x9f, 0x99, + 0xb9, 0x35, 0xb3, 0x11, 0xbe, 0xd3, 0xc3, 0x2f, 0xb2, 0xac, 0xc2, 0x8d, 0xfa, 0x82, 0x16, 0x8b, + 0x7f, 0xda, 0xb0, 0xf3, 0x5f, 0xa3, 0x07, 0x30, 0x2a, 0x72, 0xf5, 0x49, 0xe7, 0xdd, 0x0f, 0x38, + 0x3c, 0x28, 0x72, 0x75, 0x96, 0x17, 0x12, 0x25, 0xb1, 0x6a, 0x25, 0xbb, 0x93, 0xc4, 0x0a, 0xa5, + 0x43, 0x70, 0x2a, 0xf1, 0x8d, 0x39, 0x63, 0x72, 0x7d, 0x3d, 0xec, 0xc8, 0x8d, 0x42, 0x1f, 0x80, + 0x37, 0x2b, 0x1b, 0xa5, 0x99, 0x3b, 0x84, 0xb4, 0x9a, 0xe9, 0x52, 0x37, 0x05, 0xf3, 0x06, 0xbb, + 0xd4, 0x4d, 0x61, 0x80, 0x22, 0x57, 0xcc, 0x1f, 0x04, 0x8a, 0x5c, 0x21, 0x20, 0x56, 0x2c, 0x18, + 0x06, 0xc4, 0x8a, 0x3e, 0x82, 0x00, 0x67, 0xc9, 0x8a, 0x8d, 0x86, 0xa0, 0x5e, 0x8d, 0x7f, 0x10, + 0xd8, 0xc3, 0xef, 0x7d, 0x2d, 0xf4, 0x6c, 0x2e, 0x2b, 0xfa, 0xf4, 0xc6, 0x8d, 0x0f, 0x6e, 0x9c, + 0xa0, 0x63, 0x92, 0xb3, 0x8b, 0xa5, 0xbc, 0x3a, 0x33, 0xba, 0xc8, 0x1e, 0x72, 0x91, 0x73, 0xdd, + 0x45, 0x13, 0x70, 0x4d, 0x1d, 0xf5, 0xc1, 0x4e, 0x4f, 0x43, 0xcb, 0x18, 0xe0, 0x4d, 0x7a, 0x1a, + 0x12, 0x93, 0xe0, 0x69, 0x68, 0x63, 0x82, 0xa7, 0xa1, 0x73, 0xf2, 0x70, 0xfd, 0x37, 0xb2, 0xd6, + 0x9b, 0x88, 0x5c, 0x6e, 0x22, 0xf2, 0x67, 0x13, 0x91, 0xef, 0xdb, 0xc8, 0xba, 0xdc, 0x46, 0xd6, + 0xaf, 0x6d, 0x64, 0x7d, 0x08, 0x6a, 0x5d, 0x56, 0x72, 0x39, 0x9d, 0xfa, 0x68, 0xe8, 0xe7, 0xff, + 0x02, 0x00, 0x00, 0xff, 0xff, 0x61, 0x46, 0x03, 0x25, 0xfd, 0x02, 0x00, 0x00, } func (m *Label) Marshal() (dAtA []byte, err error) { @@ -353,10 +339,6 @@ func (m *Label) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Value) > 0 { i -= len(m.Value) copy(dAtA[i:], m.Value) @@ -394,10 +376,6 @@ func (m *Chunk) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Data) > 0 { i -= len(m.Data) copy(dAtA[i:], m.Data) @@ -433,10 +411,6 @@ func (m *Series) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Chunks) > 0 { for iNdEx := len(m.Chunks) - 1; iNdEx >= 0; iNdEx-- { { @@ -488,10 +462,6 @@ func (m *AggrChunk) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if m.Counter != nil { { size, err := m.Counter.MarshalToSizedBuffer(dAtA[:i]) @@ -597,10 +567,6 @@ func (m *LabelMatcher) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Value) > 0 { i -= len(m.Value) copy(dAtA[i:], m.Value) @@ -648,9 +614,6 @@ func (m *Label) Size() (n int) { if l > 0 { n += 1 + l + sovTypes(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -667,9 +630,6 @@ func (m *Chunk) Size() (n int) { if l > 0 { n += 1 + l + sovTypes(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -691,9 +651,6 @@ func (m *Series) Size() (n int) { n += 1 + l + sovTypes(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -733,9 +690,6 @@ func (m *AggrChunk) Size() (n int) { l = m.Counter.Size() n += 1 + l + sovTypes(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -756,9 +710,6 @@ func (m *LabelMatcher) Size() (n int) { if l > 0 { n += 1 + l + sovTypes(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -876,7 +827,6 @@ func (m *Label) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -983,7 +933,6 @@ func (m *Chunk) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -1105,7 +1054,6 @@ func (m *Series) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -1413,7 +1361,6 @@ func (m *AggrChunk) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -1550,7 +1497,6 @@ func (m *LabelMatcher) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } diff --git a/pkg/store/storepb/types.proto b/pkg/store/storepb/types.proto index 4c358b3cc4..24c9f4b318 100644 --- a/pkg/store/storepb/types.proto +++ b/pkg/store/storepb/types.proto @@ -10,6 +10,12 @@ option (gogoproto.marshaler_all) = true; option (gogoproto.unmarshaler_all) = true; option (gogoproto.goproto_getters_all) = false; +// Do not generate XXX fields to reduce memory footprint and opening a door +// for zero-copy casts to/from prometheus data types. +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.goproto_sizecache_all) = false; + message Label { string name = 1; string value = 2; From 2591455c47e5440e841b0b049d5b4a5d012f4dcc Mon Sep 17 00:00:00 2001 From: Martin Chodur Date: Wed, 13 Nov 2019 19:35:36 +0100 Subject: [PATCH 048/257] docs: removed old k8s demo and refer kube-thanos instead (#1600) Signed-off-by: Martin Chodur --- docs/quick-tutorial.md | 18 +- docs/release-process.md | 6 + tutorials/kubernetes-demo/.gitignore | 2 - tutorials/kubernetes-demo/README.md | 3 - .../kubernetes-demo/apply-pv-gen-metrics.sh | 17 - .../blockgen/container_mem_metrics_eu1.json | 20 - .../blockgen/container_mem_metrics_us1.json | 20 - tutorials/kubernetes-demo/blockgen/main.go | 300 ------ .../kubernetes-demo/blockgen/main_test.go | 37 - tutorials/kubernetes-demo/cluster-down.sh | 9 - tutorials/kubernetes-demo/cluster-up.sh | 20 - .../manifests/alertmanager.yaml | 122 --- .../grafana-datasources-querier.yaml | 20 - .../manifests/grafana-datasources.yaml | 29 - .../kubernetes-demo/manifests/grafana.yaml | 379 ------- .../manifests/kube-state-metrics.yaml | 169 ---- .../kubernetes-demo/manifests/minio.yaml | 46 - .../manifests/prometheus-ha-sidecar-lts.yaml | 324 ------ .../manifests/prometheus-ha-sidecar.yaml | 308 ------ .../manifests/prometheus-ha.yaml | 223 ---- .../manifests/prometheus-pv-0.yaml | 15 - .../manifests/prometheus-pv-1.yaml | 14 - .../manifests/prometheus-rules.yaml | 949 ------------------ .../kubernetes-demo/manifests/prometheus.yaml | 206 ---- .../manifests/thanos-compactor.yaml | 52 - .../manifests/thanos-querier-no-us1.yaml | 67 -- .../manifests/thanos-querier.yaml | 74 -- .../manifests/thanos-ruler.yaml | 116 --- .../manifests/thanos-store-gateway.yaml | 74 -- tutorials/kubernetes-demo/setup.sh | 47 - .../kubernetes-demo/slides/globalview-ha.svg | 1 - .../kubernetes-demo/slides/initial-setup.svg | 1 - .../slides/unlimited-retention.svg | 1 - 33 files changed, 16 insertions(+), 3673 deletions(-) delete mode 100644 tutorials/kubernetes-demo/.gitignore delete mode 100644 tutorials/kubernetes-demo/README.md delete mode 100755 tutorials/kubernetes-demo/apply-pv-gen-metrics.sh delete mode 100644 tutorials/kubernetes-demo/blockgen/container_mem_metrics_eu1.json delete mode 100644 tutorials/kubernetes-demo/blockgen/container_mem_metrics_us1.json delete mode 100644 tutorials/kubernetes-demo/blockgen/main.go delete mode 100644 tutorials/kubernetes-demo/blockgen/main_test.go delete mode 100755 tutorials/kubernetes-demo/cluster-down.sh delete mode 100755 tutorials/kubernetes-demo/cluster-up.sh delete mode 100644 tutorials/kubernetes-demo/manifests/alertmanager.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/grafana-datasources-querier.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/grafana-datasources.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/grafana.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/kube-state-metrics.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/minio.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/prometheus-ha.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/prometheus-pv-0.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/prometheus-pv-1.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/prometheus-rules.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/prometheus.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/thanos-compactor.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/thanos-querier-no-us1.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/thanos-querier.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/thanos-ruler.yaml delete mode 100644 tutorials/kubernetes-demo/manifests/thanos-store-gateway.yaml delete mode 100755 tutorials/kubernetes-demo/setup.sh delete mode 100644 tutorials/kubernetes-demo/slides/globalview-ha.svg delete mode 100644 tutorials/kubernetes-demo/slides/initial-setup.svg delete mode 100644 tutorials/kubernetes-demo/slides/unlimited-retention.svg diff --git a/docs/quick-tutorial.md b/docs/quick-tutorial.md index 437337ed38..479ba2ab83 100644 --- a/docs/quick-tutorial.md +++ b/docs/quick-tutorial.md @@ -72,8 +72,7 @@ Rolling this out has little to zero impact on the running Prometheus instance. I If you are not interested in backing up any data, the `--objstore.config-file` flag can simply be omitted. -* _[Example Kubernetes manifest](/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml)_ -* _[Example Kubernetes manifest with Minio upload](/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml)_ +* _[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)_ * _[Example Deploying sidecar using official Prometheus Helm Chart](/tutorials/kubernetes-helm/README.md)_ * _[Details & Config for other object stores](storage.md)_ @@ -92,8 +91,8 @@ thanos sidecar \ --grpc-address 0.0.0.0:19090 # GRPC endpoint for StoreAPI ``` -* _[Example Kubernetes manifest](/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml)_ -* _[Example Kubernetes manifest with GCS upload](/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml)_ +* _[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)_ + #### External Labels @@ -159,7 +158,7 @@ thanos query \ Go to the configured HTTP address, and you should now be able to query across all Prometheus instances and receive de-duplicated data. -* _[Example Kubernetes manifest](/tutorials/kubernetes-demo/manifests/thanos-querier.yaml)_ +* _[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/manifests/thanos-querier-deployment.yaml)_ #### Communication Between Components @@ -180,8 +179,7 @@ thanos query \ Read more details [here](service-discovery.md). -* _[Example Kubernetes manifest](/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml)_ -* _[Example Kubernetes manifest with GCS upload](/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml)_ +* _[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)_ ### [Store Gateway](components/store.md) @@ -199,7 +197,7 @@ thanos store \ The store gateway occupies small amounts of disk space for caching basic information about data in the object storage. This will rarely exceed more than a few gigabytes and is used to improve restart times. It is useful but not required to preserve it across restarts. -* _[Example Kubernetes manifest](/tutorials/kubernetes-demo/manifests/thanos-store-gateway.yaml)_ +* _[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/manifests/thanos-store-statefulSet.yaml)_ ### [Compactor](components/compact.md) @@ -218,7 +216,11 @@ The compactor is not in the critical path of querying or data backup. It can eit _NOTE: The compactor must be run as a **singleton** and must not run when manually modifying data in the bucket._ +* _[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/examples/all/manifests/thanos-compactor-statefulSet.yaml)_ + ### [Ruler/Rule](components/rule.md) In case of Prometheus with Thanos sidecar does not have enough retention, or if you want to have alerts or recording rules that requires global view, Thanos has just the component for that: the [Ruler](components/rule.md), which does rule and alert evaluation on top of a given Thanos Querier. + + diff --git a/docs/release-process.md b/docs/release-process.md index 7f9e33ab92..6b25b89abf 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -62,6 +62,12 @@ Process of releasing a *minor* Thanos version: 1. *In case of version after `v1+.y.z`*, double check if none of the changes break API compatibility. This should be done in PR review process, but double check is good to have. 1. In case of `v0.y.z`, document all incompatibilities in changelog. +1. Update tutorials: + 1. Update the Thanos version used in the [tutorials](../tutorials) manifests. + 1. In case of any breaking changes or necessary updates adjust the manifests + so the tutorial stays up to date. + 1. Update the [scripts/quickstart.sh](../scripts/quickstart.sh) script if needed. + 1. After review, merge the PR and immediately after this tag a version: ```bash diff --git a/tutorials/kubernetes-demo/.gitignore b/tutorials/kubernetes-demo/.gitignore deleted file mode 100644 index ed5eee1a37..0000000000 --- a/tutorials/kubernetes-demo/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -blockgen/out/* -.demo-last-step \ No newline at end of file diff --git a/tutorials/kubernetes-demo/README.md b/tutorials/kubernetes-demo/README.md deleted file mode 100644 index 3755030d9d..0000000000 --- a/tutorials/kubernetes-demo/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Thanos - Transforming Prometheus to a Global Scale in a Seven Simple Steps - -TODO: Describe full demo. diff --git a/tutorials/kubernetes-demo/apply-pv-gen-metrics.sh b/tutorials/kubernetes-demo/apply-pv-gen-metrics.sh deleted file mode 100755 index 362f2cf461..0000000000 --- a/tutorials/kubernetes-demo/apply-pv-gen-metrics.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -cluster=$1 -replica=$2 -retention=$3 - -# Add volume and generated metrics inside it. -kubectl apply --context=${cluster} -f manifests/prometheus-pv-${replica}.yaml - -rm -rf -- /tmp/prom-out -mkdir /tmp/prom-out -go run ./blockgen/main.go --input=./blockgen/container_mem_metrics_${cluster}.json --output-dir=/tmp/prom-out --retention=${retention} -chmod -R 775 /tmp/prom-out -# Fun with permissions because Prometheus process is run a "noone" in a pod... ): -minikube -p ${cluster} ssh "sudo rm -rf /data/pv-prometheus-${replica} && sudo mkdir /data/pv-prometheus-${replica} && sudo chmod -R 777 /data/pv-prometheus-${replica}" -scp -r -i $(minikube -p ${cluster} ssh-key) /tmp/prom-out/* docker@$(minikube -p ${cluster} ip):/data/pv-prometheus-${replica}/ -minikube -p ${cluster} ssh "sudo chmod -R 777 /data/pv-prometheus-${replica}" \ No newline at end of file diff --git a/tutorials/kubernetes-demo/blockgen/container_mem_metrics_eu1.json b/tutorials/kubernetes-demo/blockgen/container_mem_metrics_eu1.json deleted file mode 100644 index a99bc78951..0000000000 --- a/tutorials/kubernetes-demo/blockgen/container_mem_metrics_eu1.json +++ /dev/null @@ -1,20 +0,0 @@ -[ { - "type": "gauge", - "changeInterval": "1h", - "jitter": 30000000, - "max": 200000000, - "min": 10000000, - "result": {"resultType":"vector","result":[{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"default","pod":"alertmanager-0"},"value":[1548809388.643,"483328"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"default","pod":"grafana-554c585fc6-2p7gg"},"value":[1548809388.643,"491520"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"default","pod":"prometheus-0"},"value":[1548809388.643,"462848"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"503808"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"434176"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"etcd-minikube"},"value":[1548809388.643,"454656"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-addon-manager-minikube"},"value":[1548809388.643,"491520"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-apiserver-minikube"},"value":[1548809388.643,"430080"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-controller-manager-minikube"},"value":[1548809388.643,"1257472"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-proxy-6mn4k"},"value":[1548809388.643,"487424"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-scheduler-minikube"},"value":[1548809388.643,"442368"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"466944"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kubernetes-dashboard-ccc79bfc9-ckbkr"},"value":[1548809388.643,"397312"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"storage-provisioner"},"value":[1548809388.643,"438272"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"addon-resizer","image":"sha256:b57c00a12f6cf8acf10de9c5e2c5adacbf355b181dd76f4d65bcfd3a936ea289","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"7045120"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"alertmanager","image":"sha256:23744b2d645c0574015adfba4a90283b79251aee3169dbe67f335d8465a8a63f","instance":"minikube","job":"kubelet","namespace":"default","pod":"alertmanager-0"},"value":[1548809388.643,"11284480"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"configmap-reload","image":"sha256:b70d7dba98e65dd440be03a3b8f28bbdd05ca0d55f6ab16f90292cd22cb961c5","instance":"minikube","job":"kubelet","namespace":"default","pod":"alertmanager-0"},"value":[1548809388.643,"1327104"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"coredns","image":"sha256:f59dcacceff45b5474d1385cd5f500d0c019ed9ca50ed5b814ac0c5fcec8699e","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"33697792"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"coredns","image":"sha256:f59dcacceff45b5474d1385cd5f500d0c019ed9ca50ed5b814ac0c5fcec8699e","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"18620416"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"etcd","image":"sha256:3cab8e1b9802cbe23a2703c2750ac4baa90b049b65e2a9e0a83e9e2c29f0724f","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"etcd-minikube"},"value":[1548809388.643,"132898816"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"grafana","image":"sha256:920eb69ade2a293782a87bc56f0b68aadb9fd0989b96f9bc88d88981aea380a1","instance":"minikube","job":"kubelet","namespace":"default","pod":"grafana-554c585fc6-2p7gg"},"value":[1548809388.643,"38207488"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-addon-manager","image":"sha256:9c16409588eb19394b90703bdb5bcfb7c08fe75308a5db30b95ca8f6bd6bdc85","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-addon-manager-minikube"},"value":[1548809388.643,"42958848"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-apiserver","image":"sha256:177db4b8e93a6a74ab19435edf17111d3ad18a8a4efef728712ea067ea8047c1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-apiserver-minikube"},"value":[1548809388.643,"374333440"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-controller-manager","image":"sha256:b9027a78d94c15a4aba54d45476c6f295c0db8f9dcb6fca34c8beff67d90a374","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-controller-manager-minikube"},"value":[1548809388.643,"112422912"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-proxy","image":"sha256:01cfa56edcfc350d36cea9c2fc857949b36bc69bf69df6901e0fd9be3c826617","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-proxy-6mn4k"},"value":[1548809388.643,"17539072"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-scheduler","image":"sha256:3193be46e0b3e215877b122052c0c7d3ef0902cf1dd6efaf3db95f37cf697002","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-scheduler-minikube"},"value":[1548809388.643,"41316352"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-state-metrics","image":"sha256:91599517197a204c99cd2c7e2175c25e18d82f9b53fc9d86f7d9976a3a6c6521","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"8130560"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kubernetes-dashboard","image":"sha256:f9aed6605b814b69e92dece6a50ed1e4e730144eb1cc971389dde9cb3820d124","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kubernetes-dashboard-ccc79bfc9-ckbkr"},"value":[1548809388.643,"12353536"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"prometheus","image":"sha256:5517f7057e7295a89a67c3c4869d60e019526d3d3ac0e45ae2e48c949b5c3f78","instance":"minikube","job":"kubelet","namespace":"default","pod":"prometheus-0"},"value":[1548809388.643,"215818240"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"storage-provisioner","image":"sha256:4689081edb103a9e8174bf23a255bfbe0b2d9ed82edc907abab6989d1c60f02c","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"storage-provisioner"},"value":[1548809388.643,"17342464"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet"},"value":[1548809388.643,"2690859008"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"default","pod":"alertmanager-0"},"value":[1548809388.643,"13094912"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"default","pod":"grafana-554c585fc6-2p7gg"},"value":[1548809388.643,"38690816"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"default","pod":"prometheus-0"},"value":[1548809388.643,"216293376"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"34201600"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"19054592"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"etcd-minikube"},"value":[1548809388.643,"133349376"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-addon-manager-minikube"},"value":[1548809388.643,"43450368"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-apiserver-minikube"},"value":[1548809388.643,"374763520"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-controller-manager-minikube"},"value":[1548809388.643,"113680384"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-proxy-6mn4k"},"value":[1548809388.643,"18026496"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-scheduler-minikube"},"value":[1548809388.643,"41758720"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"15642624"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kubernetes-dashboard-ccc79bfc9-ckbkr"},"value":[1548809388.643,"30040064"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"storage-provisioner"},"value":[1548809388.643,"65912832"]}]} -},{ - "type": "gauge", - "jitter": 0, - "max": 200000000, - "min": 100000000, - "result": {"resultType":"vector","result":[{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"eu1","container":"addon-resizer","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"52428800"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"eu1","container":"alertmanager","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"default","node":"minikube","pod":"alertmanager-0"},"value":[1548809388.643,"209715200"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"eu1","container":"configmap-reload","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"default","node":"minikube","pod":"alertmanager-0"},"value":[1548809388.643,"10485760"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"eu1","container":"coredns","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"73400320"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"eu1","container":"coredns","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"73400320"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"eu1","container":"grafana","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"default","node":"minikube","pod":"grafana-554c585fc6-2p7gg"},"value":[1548809388.643,"209715200"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"eu1","container":"kube-addon-manager","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-addon-manager-minikube"},"value":[1548809388.643,"52428800"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"eu1","container":"kube-state-metrics","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"106954752"]}]} -}, { - "type": "gauge", - "jitter": 0, - "max": 200000000, - "min": 100000000, - "result": {"resultType":"vector","result":[{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"eu1","container":"addon-resizer","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"52428800"]},{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"eu1","container":"alertmanager","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"default","node":"minikube","pod":"alertmanager-0"},"value":[1548809388.643,"209715200"]},{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"eu1","container":"configmap-reload","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"default","node":"minikube","pod":"alertmanager-0"},"value":[1548809388.643,"10485760"]},{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"eu1","container":"coredns","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"178257920"]},{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"eu1","container":"coredns","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"178257920"]},{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"eu1","container":"grafana","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"default","node":"minikube","pod":"grafana-554c585fc6-2p7gg"},"value":[1548809388.643,"209715200"]},{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"eu1","container":"kube-state-metrics","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"106954752"]}]} -}] \ No newline at end of file diff --git a/tutorials/kubernetes-demo/blockgen/container_mem_metrics_us1.json b/tutorials/kubernetes-demo/blockgen/container_mem_metrics_us1.json deleted file mode 100644 index 86c4cccb8c..0000000000 --- a/tutorials/kubernetes-demo/blockgen/container_mem_metrics_us1.json +++ /dev/null @@ -1,20 +0,0 @@ -[ { - "type": "gauge", - "changeInterval": "1h", - "jitter": 30000000, - "max": 200000000, - "min": 10000000, - "result": {"resultType":"vector","result": [{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"default","pod":"prometheus-0"},"value":[1548809388.643,"462848"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"503808"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"434176"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"etcd-minikube"},"value":[1548809388.643,"454656"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-addon-manager-minikube"},"value":[1548809388.643,"491520"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-apiserver-minikube"},"value":[1548809388.643,"430080"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-controller-manager-minikube"},"value":[1548809388.643,"1257472"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-proxy-6mn4k"},"value":[1548809388.643,"487424"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-scheduler-minikube"},"value":[1548809388.643,"442368"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"466944"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kubernetes-dashboard-ccc79bfc9-ckbkr"},"value":[1548809388.643,"397312"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"POD","image":"k8s.gcr.io/pause:3.1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"storage-provisioner"},"value":[1548809388.643,"438272"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"addon-resizer","image":"sha256:b57c00a12f6cf8acf10de9c5e2c5adacbf355b181dd76f4d65bcfd3a936ea289","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"7045120"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"coredns","image":"sha256:f59dcacceff45b5474d1385cd5f500d0c019ed9ca50ed5b814ac0c5fcec8699e","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"33697792"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"coredns","image":"sha256:f59dcacceff45b5474d1385cd5f500d0c019ed9ca50ed5b814ac0c5fcec8699e","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"18620416"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"etcd","image":"sha256:3cab8e1b9802cbe23a2703c2750ac4baa90b049b65e2a9e0a83e9e2c29f0724f","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"etcd-minikube"},"value":[1548809388.643,"132898816"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-addon-manager","image":"sha256:9c16409588eb19394b90703bdb5bcfb7c08fe75308a5db30b95ca8f6bd6bdc85","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-addon-manager-minikube"},"value":[1548809388.643,"42958848"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-apiserver","image":"sha256:177db4b8e93a6a74ab19435edf17111d3ad18a8a4efef728712ea067ea8047c1","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-apiserver-minikube"},"value":[1548809388.643,"374333440"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-controller-manager","image":"sha256:b9027a78d94c15a4aba54d45476c6f295c0db8f9dcb6fca34c8beff67d90a374","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-controller-manager-minikube"},"value":[1548809388.643,"112422912"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-proxy","image":"sha256:01cfa56edcfc350d36cea9c2fc857949b36bc69bf69df6901e0fd9be3c826617","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-proxy-6mn4k"},"value":[1548809388.643,"17539072"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-scheduler","image":"sha256:3193be46e0b3e215877b122052c0c7d3ef0902cf1dd6efaf3db95f37cf697002","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-scheduler-minikube"},"value":[1548809388.643,"41316352"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kube-state-metrics","image":"sha256:91599517197a204c99cd2c7e2175c25e18d82f9b53fc9d86f7d9976a3a6c6521","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"8130560"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"kubernetes-dashboard","image":"sha256:f9aed6605b814b69e92dece6a50ed1e4e730144eb1cc971389dde9cb3820d124","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kubernetes-dashboard-ccc79bfc9-ckbkr"},"value":[1548809388.643,"12353536"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"prometheus","image":"sha256:5517f7057e7295a89a67c3c4869d60e019526d3d3ac0e45ae2e48c949b5c3f78","instance":"minikube","job":"kubelet","namespace":"default","pod":"prometheus-0"},"value":[1548809388.643,"215818240"]},{"metric":{"__name__":"container_memory_usage_bytes","container_name":"storage-provisioner","image":"sha256:4689081edb103a9e8174bf23a255bfbe0b2d9ed82edc907abab6989d1c60f02c","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"storage-provisioner"},"value":[1548809388.643,"17342464"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet"},"value":[1548809388.643,"2690859008"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"default","pod":"prometheus-0"},"value":[1548809388.643,"216293376"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"34201600"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"19054592"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"etcd-minikube"},"value":[1548809388.643,"133349376"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-addon-manager-minikube"},"value":[1548809388.643,"43450368"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-apiserver-minikube"},"value":[1548809388.643,"374763520"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-controller-manager-minikube"},"value":[1548809388.643,"113680384"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-proxy-6mn4k"},"value":[1548809388.643,"18026496"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-scheduler-minikube"},"value":[1548809388.643,"41758720"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"15642624"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"kubernetes-dashboard-ccc79bfc9-ckbkr"},"value":[1548809388.643,"30040064"]},{"metric":{"__name__":"container_memory_usage_bytes","instance":"minikube","job":"kubelet","namespace":"kube-system","pod":"storage-provisioner"},"value":[1548809388.643,"65912832"]}]} -},{ - "type": "gauge", - "jitter": 0, - "max": 200000000, - "min": 100000000, - "result": {"resultType":"vector","result":[{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"us1","container":"addon-resizer","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"52428800"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"us1","container":"coredns","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"73400320"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"us1","container":"coredns","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"73400320"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"us1","container":"kube-addon-manager","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-addon-manager-minikube"},"value":[1548809388.643,"52428800"]},{"metric":{"__name__":"kube_pod_container_resource_requests_memory_bytes","cluster":"us1","container":"kube-state-metrics","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"106954752"]}]} -}, { - "type": "gauge", - "jitter": 0, - "max": 200000000, - "min": 100000000, - "result": {"resultType":"vector","result":[{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"us1","container":"addon-resizer","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"52428800"]},{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"us1","container":"coredns","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"coredns-86c58d9df4-262fj"},"value":[1548809388.643,"178257920"]},{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"us1","container":"coredns","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"coredns-86c58d9df4-xv6ts"},"value":[1548809388.643,"178257920"]},{"metric":{"__name__":"kube_pod_container_resource_limits_memory_bytes","cluster":"us1","container":"kube-state-metrics","instance":"172.17.0.9:8080","job":"kube-state-metrics","namespace":"kube-system","node":"minikube","pod":"kube-state-metrics-68f6cc566c-vp566"},"value":[1548809388.643,"106954752"]}]} -}] \ No newline at end of file diff --git a/tutorials/kubernetes-demo/blockgen/main.go b/tutorials/kubernetes-demo/blockgen/main.go deleted file mode 100644 index ece4ddaf18..0000000000 --- a/tutorials/kubernetes-demo/blockgen/main.go +++ /dev/null @@ -1,300 +0,0 @@ -package main - -import ( - "encoding/json" - "io/ioutil" - "math/rand" - "os" - "path/filepath" - "strings" - "time" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pkg/timestamp" - "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/tsdb/labels" - "gopkg.in/alecthomas/kingpin.v2" -) - -// Allow for more realistic output. -type series struct { - Type string // gauge, counter (if conunter we treat below as rate aim). - Jitter float64 - ChangeInterval string - Max float64 - Min float64 - Result queryData -} - -type queryData struct { - ResultType model.ValueType `json:"resultType"` - Result model.Vector `json:"result"` -} - -func main() { - app := kingpin.New(filepath.Base(os.Args[0]), "Generates artificial metrics from min time to given max time in compacted TSDB format (including head WAL).") - app.HelpFlag.Short('h') - input := app.Flag("input", "Input file for series config.").Required().String() - outputDir := app.Flag("output-dir", "Output directory for generated TSDB data.").Required().String() - scrapeInterval := app.Flag("scrape-interval", "Interval for to generate samples with.").Default("15s").Duration() - - retention := app.Flag("retention", "Defines the max time in relation to current time for generated samples.").Required().Duration() - - logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) - _, err := app.Parse(os.Args[1:]) - if err != nil { - level.Error(logger).Log("err", err) - os.Exit(1) - } - - f, err := ioutil.ReadFile(*input) - if err != nil { - level.Error(logger).Log("err", err, "file", input) - os.Exit(1) - } - - var s []series - if err := json.Unmarshal(f, &s); err != nil { - level.Error(logger).Log("err", err) - os.Exit(1) - } - - // Same code as Prometheus for compaction levels and max block. - rngs := tsdb.ExponentialBlockRanges(int64(time.Duration(2*time.Hour).Seconds()*1000), 10, 3) - maxBlockDuration := *retention / 10 - for i, v := range rngs { - if v > int64(maxBlockDuration.Seconds()*1000) { - rngs = rngs[:i] - break - } - } - - if len(rngs) == 0 { - rngs = append(rngs, int64(time.Duration(2*time.Hour).Seconds()*1000)) - } - - if err := os.RemoveAll(*outputDir); err != nil { - level.Error(logger).Log("msg", "remove output dir", "err", err) - os.Exit(1) - } - - db, err := tsdb.Open(*outputDir, nil, nil, &tsdb.Options{ - BlockRanges: rngs, - RetentionDuration: uint64(retention.Seconds() * 1000), - NoLockfile: true, - }) - if err != nil { - level.Error(logger).Log("err", err) - os.Exit(1) - } - - // Of course there will be small gap in minTime vs time.Now once we finish. - // We are fine with this. - n := time.Now() - maxTime := timestamp.FromTime(n) - minTime := timestamp.FromTime(n.Add(-*retention)) - - generators := make(map[string]gen) - for _, in := range s { - for _, r := range in.Result.Result { - lset := labels.New() - for n, v := range r.Metric { - lset = append(lset, labels.Label{Name: string(n), Value: string(v)}) - } - var chInterval time.Duration - if in.ChangeInterval != "" { - chInterval, err = time.ParseDuration(in.ChangeInterval) - if err != nil { - level.Error(logger).Log("err", err) - os.Exit(1) - } - } - - switch strings.ToLower(in.Type) { - case "counter": - // Does not work well (: Too naive. - generators[lset.String()] = &counterGen{ - interval: *scrapeInterval, - maxTime: maxTime, - minTime: minTime, - lset: lset, - min: in.Min, - max: in.Max, - jitter: in.Jitter, - rateInterval: 5 * time.Minute, - changeInterval: chInterval, - } - case "gauge": - generators[lset.String()] = &gaugeGen{ - interval: *scrapeInterval, - maxTime: maxTime, - minTime: minTime, - lset: lset, - min: in.Min, - max: in.Max, - jitter: in.Jitter, - changeInterval: chInterval, - } - default: - level.Error(logger).Log("msg", "unknown metric type", "type", in.Type) - os.Exit(1) - } - } - } - - a := db.Appender() - for _, generator := range generators { - for generator.Next() { - // Cache reference and use AddFast if we are too slow. - if _, err := a.Add(generator.Lset(), generator.Ts(), generator.Value()); err != nil { - level.Error(logger).Log("msg", "add", "err", err) - os.Exit(1) - } - } - } - - if err := a.Commit(); err != nil { - level.Error(logger).Log("msg", "commit", "err", err) - os.Exit(1) - } - - // Don't wait for compact, it will be compacted by Prometheus anyway. - - if err := db.Close(); err != nil { - level.Error(logger).Log("msg", "close", "err", err) - os.Exit(1) - } - - level.Info(logger).Log("msg", "generated artificial metrics", "series", len(generators)) -} - -type gaugeGen struct { - changeInterval time.Duration - interval time.Duration - maxTime, minTime int64 - - lset labels.Labels - min, max, jitter float64 - - v float64 - mod float64 - init bool - elapsed int64 -} - -func (g *gaugeGen) Lset() labels.Labels { - return g.lset -} - -func (g *gaugeGen) Next() bool { - if g.minTime > g.maxTime { - return false - } - defer func() { - g.minTime += int64(g.interval.Seconds() * 1000) - g.elapsed += int64(g.interval.Seconds() * 1000) - }() - - if !g.init { - g.v = g.min + rand.Float64()*((g.max-g.min)+1) - g.init = true - } - - // Technically only mod changes. - if g.jitter > 0 && g.elapsed >= int64(g.changeInterval.Seconds()*1000) { - g.mod = (rand.Float64() - 0.5) * g.jitter - g.elapsed = 0 - } - - return true -} - -func (g *gaugeGen) Ts() int64 { return g.minTime } -func (g *gaugeGen) Value() float64 { return g.v + g.mod } - -type counterGen struct { - maxTime, minTime int64 - - lset labels.Labels - min, max, jitter float64 - interval time.Duration - changeInterval time.Duration - rateInterval time.Duration - - v float64 - init bool - buff []promql.Point - - lastVal float64 - elapsed int64 -} - -func (g *counterGen) Lset() labels.Labels { - return g.lset -} - -func (g *counterGen) Next() bool { - defer func() { g.elapsed += int64(g.interval.Seconds() * 1000) }() - - if g.init && len(g.buff) == 0 { - return false - } - - if len(g.buff) > 0 { - // Pop front. - g.buff = g.buff[1:] - - if len(g.buff) > 0 { - return true - } - } - - if !g.init { - g.v = g.min + rand.Float64()*((g.max-g.min)+1) - g.init = true - } - - var mod float64 - if g.jitter > 0 && g.elapsed >= int64(g.changeInterval.Seconds()*1000) { - mod = (rand.Float64() - 0.5) * g.jitter - - if mod > g.v { - mod = g.v - } - - g.elapsed = 0 - } - - // Distribute goalV into multiple rateInterval/interval increments. - comps := make([]float64, int64(g.rateInterval/g.interval)) - var sum float64 - for i := range comps { - comps[i] = rand.Float64() - sum += comps[i] - } - - // That's the goal for our rate. - x := g.v + mod/sum - for g.minTime <= g.maxTime && len(comps) > 0 { - g.lastVal += x * comps[0] - comps = comps[1:] - - g.minTime += int64(g.interval.Seconds() * 1000) - g.buff = append(g.buff, promql.Point{T: g.minTime, V: g.lastVal}) - } - - return len(g.buff) > 0 -} - -func (g *counterGen) Ts() int64 { return g.buff[0].T } -func (g *counterGen) Value() float64 { return g.buff[0].V } - -type gen interface { - Lset() labels.Labels - Next() bool - Ts() int64 - Value() float64 -} diff --git a/tutorials/kubernetes-demo/blockgen/main_test.go b/tutorials/kubernetes-demo/blockgen/main_test.go deleted file mode 100644 index a574057387..0000000000 --- a/tutorials/kubernetes-demo/blockgen/main_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "testing" - "time" - - "github.com/thanos-io/thanos/pkg/testutil" -) - -func TestCounterGen(t *testing.T) { - g := &counterGen{ - minTime: 100, - maxTime: int64((24 * time.Hour).Seconds()) * 1000, - interval: 15 * time.Second, - rateInterval: 5 * time.Minute, - min: 100, - max: 400, - jitter: 300, - } - - lastV := float64(0) - lastT := int64(0) - - init := false - samples := int64(0) - for g.Next() { - samples++ - if init { - testutil.Assert(t, lastV <= g.Value(), "") - testutil.Assert(t, lastT <= g.Ts(), "") - init = true - } - lastV = g.Value() - lastT = g.Ts() - } - testutil.Equals(t, int64((24*time.Hour)/(15*time.Second)), samples) -} diff --git a/tutorials/kubernetes-demo/cluster-down.sh b/tutorials/kubernetes-demo/cluster-down.sh deleted file mode 100755 index d63a5d5861..0000000000 --- a/tutorials/kubernetes-demo/cluster-down.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -e - -minikube -p us1 stop -minikube -p us1 delete - -minikube -p eu1 stop -minikube -p eu1 delete diff --git a/tutorials/kubernetes-demo/cluster-up.sh b/tutorials/kubernetes-demo/cluster-up.sh deleted file mode 100755 index c323578a08..0000000000 --- a/tutorials/kubernetes-demo/cluster-up.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -set -e - -minikube start --cache-images --vm-driver=kvm2 -p us1 --kubernetes-version="v1.13.2" \ - --memory=8192 --cpus=4 \ - --extra-config=kubelet.authentication-token-webhook=true \ - --extra-config=kubelet.authorization-mode=Webhook \ - --extra-config=scheduler.address=0.0.0.0 \ - --extra-config=controller-manager.address=0.0.0.0 - -minikube start --cache-images --vm-driver=kvm2 -p eu1 --kubernetes-version="v1.13.2" \ - --memory=8192 --cpus=4 \ - --extra-config=kubelet.authentication-token-webhook=true \ - --extra-config=kubelet.authorization-mode=Webhook \ - --extra-config=scheduler.address=0.0.0.0 \ - --extra-config=controller-manager.address=0.0.0.0 - -ssh-keyscan $(minikube -p eu1 ip) >> ~/.ssh/known_hosts -ssh-keyscan $(minikube -p us1 ip) >> ~/.ssh/known_hosts \ No newline at end of file diff --git a/tutorials/kubernetes-demo/manifests/alertmanager.yaml b/tutorials/kubernetes-demo/manifests/alertmanager.yaml deleted file mode 100644 index f281f0f7e7..0000000000 --- a/tutorials/kubernetes-demo/manifests/alertmanager.yaml +++ /dev/null @@ -1,122 +0,0 @@ -kind: PersistentVolume -apiVersion: v1 -metadata: - name: pv-alertmanager-0 - labels: - type: local -spec: - storageClassName: alert-manual - capacity: - storage: 5Gi - accessModes: - - ReadWriteOnce - hostPath: - path: "/data/pv-alertmanager-0" ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: alertmanager - labels: - app: alertmanager -spec: - serviceName: "alertmanager" - replicas: 1 - selector: - matchLabels: - app: alertmanager - template: - metadata: - labels: - app: alertmanager - spec: - containers: - - name: alertmanager - image: "prom/alertmanager:v0.14.0" - args: - - --config.file=/etc/config/alertmanager.yml - - --storage.path=/data - - --web.external-url=/ - ports: - - containerPort: 9093 - name: http - readinessProbe: - httpGet: - path: "/#/status" - port: 9093 - initialDelaySeconds: 30 - timeoutSeconds: 30 - volumeMounts: - - name: config-volume - mountPath: /etc/config - - name: alertmanager - mountPath: "/data" - subPath: "" - resources: - limits: - cpu: 200m - memory: 200Mi - requests: - cpu: 200m - memory: 200Mi - - name: configmap-reload - image: "jimmidyson/configmap-reload:v0.1" - args: - - --volume-dir=/etc/config - - --webhook-url=http://localhost:9093/-/reload - volumeMounts: - - name: config-volume - mountPath: /etc/config - readOnly: true - resources: - limits: - cpu: 10m - memory: 10Mi - requests: - cpu: 10m - memory: 10Mi - volumes: - - name: config-volume - configMap: - name: alertmanager - volumeClaimTemplates: - - metadata: - labels: - app: alertmanager - name: alertmanager - spec: - storageClassName: alert-manual - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi ---- -apiVersion: v1 -kind: Service -metadata: - name: alertmanager -spec: - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 9093 - selector: - app: alertmanager - type: NodePort ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: alertmanager -data: - alertmanager.yml: | - global: null - receivers: - - name: default-receiver - route: - group_interval: 5m - group_wait: 10s - receiver: default-receiver - repeat_interval: 3h \ No newline at end of file diff --git a/tutorials/kubernetes-demo/manifests/grafana-datasources-querier.yaml b/tutorials/kubernetes-demo/manifests/grafana-datasources-querier.yaml deleted file mode 100644 index 8c5c56f2c1..0000000000 --- a/tutorials/kubernetes-demo/manifests/grafana-datasources-querier.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -data: - prometheus.yaml: |- - { - "apiVersion": 1, - "datasources": [ - { - "access": "proxy", - "editable": false, - "name": "all", - "orgId": 1, - "type": "prometheus", - "url": "http://thanos-querier.default.svc:9090", - "version": 1 - }, - ] - } -kind: ConfigMap -metadata: - name: grafana-datasources \ No newline at end of file diff --git a/tutorials/kubernetes-demo/manifests/grafana-datasources.yaml b/tutorials/kubernetes-demo/manifests/grafana-datasources.yaml deleted file mode 100644 index 3a738df7d5..0000000000 --- a/tutorials/kubernetes-demo/manifests/grafana-datasources.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v1 -data: - prometheus.yaml: |- - { - "apiVersion": 1, - "datasources": [ - { - "access": "proxy", - "editable": false, - "name": "eu1", - "orgId": 1, - "type": "prometheus", - "url": "http://prometheus.default.svc:9090", - "version": 1 - }, - { - "access": "proxy", - "editable": false, - "name": "us1", - "orgId": 1, - "type": "prometheus", - "url": "%%PROM_US1_URL%%", - "version": 1 - } - ] - } -kind: ConfigMap -metadata: - name: grafana-datasources \ No newline at end of file diff --git a/tutorials/kubernetes-demo/manifests/grafana.yaml b/tutorials/kubernetes-demo/manifests/grafana.yaml deleted file mode 100644 index b9d59b5f0b..0000000000 --- a/tutorials/kubernetes-demo/manifests/grafana.yaml +++ /dev/null @@ -1,379 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: grafana - name: grafana -spec: - replicas: 1 - selector: - matchLabels: - app: grafana - template: - metadata: - labels: - app: grafana - spec: - containers: - - image: grafana/grafana:5.2.4 - name: grafana - ports: - - containerPort: 3000 - name: http - readinessProbe: - httpGet: - path: /api/health - port: http - resources: - limits: - cpu: 200m - memory: 200Mi - requests: - cpu: 200m - memory: 200Mi - volumeMounts: - - mountPath: /var/lib/grafana - name: grafana-storage - - mountPath: /etc/grafana/provisioning/datasources - readOnly: false - name: grafana-datasources - - mountPath: /etc/grafana/provisioning/dashboards - name: grafana-dashboards - - mountPath: /grafana-dashboard-definitions/0/pods-memory - name: grafana-pods-memory - env: - - name: GF_AUTH_ANONYMOUS_ENABLED - value: "true" - - name: GF_AUTH_ANONYMOUS_ORG_ROLE - value: "Admin" - securityContext: - runAsNonRoot: true - runAsUser: 65534 - serviceAccountName: grafana - volumes: - - emptyDir: {} - name: grafana-storage - - name: grafana-datasources - configMap: - name: grafana-datasources - - configMap: - name: grafana-dashboards - name: grafana-dashboards - - configMap: - name: grafana-pods-memory - name: grafana-pods-memory ---- -apiVersion: v1 -data: - dashboards.yaml: |- - { - "apiVersion": 1, - "providers": [ - { - "folder": "", - "name": "0", - "options": { - "path": "/grafana-dashboard-definitions/0" - }, - "orgId": 1, - "type": "file" - } - ] - } -kind: ConfigMap -metadata: - name: grafana-dashboards ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: grafana ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: grafana - name: grafana -spec: - ports: - - name: http - port: 3000 - targetPort: http - selector: - app: grafana - type: NodePort ---- -apiVersion: v1 -items: - - apiVersion: v1 - data: - pod.json: |- - { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "iteration": 1548898867144, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(container_name, cluster) (container_memory_usage_bytes{job=\"kubelet\", namespace=\"$namespace\", pod=\"$pod\", container_name=~\"$container\", container_name!=\"POD\", cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Current: {{ container_name }} {{ cluster }}", - "refId": "A" - }, - { - "expr": "sum by(container, cluster) (kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Requested: {{ container }} {{ cluster }}", - "refId": "B" - }, - { - "expr": "sum by(container, cluster) (kube_pod_container_resource_limits_memory_bytes{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Limit: {{ container }} {{ cluster }}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "eu1", - "value": "eu1" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": ".*", - "current": {}, - "datasource": "$datasource", - "hide": 0, - "includeAll": true, - "label": "Cluster", - "multi": true, - "name": "cluster", - "options": [], - "query": "label_values(container_memory_usage_bytes, cluster)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "text": "default", - "value": "default" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": "label_values(container_memory_usage_bytes, namespace)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "text": "prometheus-0", - "value": "prometheus-0" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Pod", - "multi": false, - "name": "pod", - "options": [], - "query": "label_values(container_memory_usage_bytes{namespace=~\"$namespace\"}, pod)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "text": "prometheus", - "value": "prometheus" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": true, - "label": "Container", - "multi": false, - "name": "container", - "options": [], - "query": "label_values(container_memory_usage_bytes{namespace=\"$namespace\", pod=\"$pod\", container_name!=\"POD\"}, container_name)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1d", - "to": "now-1h" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Pods Memory", - "uid": "pods_memory", - "version": 1202 - } - kind: ConfigMap - metadata: - name: grafana-pods-memory -kind: ConfigMapList diff --git a/tutorials/kubernetes-demo/manifests/kube-state-metrics.yaml b/tutorials/kubernetes-demo/manifests/kube-state-metrics.yaml deleted file mode 100644 index 6685433af8..0000000000 --- a/tutorials/kubernetes-demo/manifests/kube-state-metrics.yaml +++ /dev/null @@ -1,169 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kube-state-metrics - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: kube-state-metrics - replicas: 1 - template: - metadata: - labels: - k8s-app: kube-state-metrics - spec: - serviceAccountName: kube-state-metrics - containers: - - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.5.0 - ports: - - name: http - containerPort: 8080 - - name: telemetry - containerPort: 8081 - readinessProbe: - httpGet: - path: /healthz - port: 8080 - initialDelaySeconds: 5 - timeoutSeconds: 5 - - name: addon-resizer - image: k8s.gcr.io/addon-resizer:1.8.3 - resources: - limits: - cpu: 150m - memory: 50Mi - requests: - cpu: 150m - memory: 50Mi - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /pod_nanny - - --container=kube-state-metrics - - --cpu=100m - - --extra-cpu=1m - - --memory=100Mi - - --extra-memory=2Mi - - --threshold=5 - - --deployment=kube-state-metrics ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-state-metrics - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: kube-state-metrics -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kube-state-metrics -subjects: - - kind: ServiceAccount - name: kube-state-metrics - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: kube-state-metrics -rules: - - apiGroups: [""] - resources: - - configmaps - - secrets - - nodes - - pods - - services - - resourcequotas - - replicationcontrollers - - limitranges - - persistentvolumeclaims - - persistentvolumes - - namespaces - - endpoints - verbs: ["list", "watch"] - - apiGroups: ["extensions"] - resources: - - daemonsets - - deployments - - replicasets - verbs: ["list", "watch"] - - apiGroups: ["apps"] - resources: - - statefulsets - verbs: ["list", "watch"] - - apiGroups: ["batch"] - resources: - - cronjobs - - jobs - verbs: ["list", "watch"] - - apiGroups: ["autoscaling"] - resources: - - horizontalpodautoscalers - verbs: ["list", "watch"] - - apiGroups: ["policy"] - resources: - - poddisruptionbudgets - verbs: ["list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: kube-state-metrics - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kube-state-metrics-resizer -subjects: - - kind: ServiceAccount - name: kube-state-metrics - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - namespace: kube-system - name: kube-state-metrics-resizer -rules: - - apiGroups: [""] - resources: - - pods - verbs: ["get"] - - apiGroups: ["extensions"] - resources: - - deployments - resourceNames: ["kube-state-metrics"] - verbs: ["get", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: kube-state-metrics - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kube-state-metrics-resizer -subjects: - - kind: ServiceAccount - name: kube-state-metrics - namespace: kube-system \ No newline at end of file diff --git a/tutorials/kubernetes-demo/manifests/minio.yaml b/tutorials/kubernetes-demo/manifests/minio.yaml deleted file mode 100644 index 40a17ed52d..0000000000 --- a/tutorials/kubernetes-demo/manifests/minio.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: minio - name: minio -spec: - ports: - - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - statefulset.kubernetes.io/pod-name: minio-0 - type: NodePort ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: minio - labels: - app: minio -spec: - serviceName: "minio" - replicas: 1 - selector: - matchLabels: - app: minio - template: - metadata: - labels: - app: minio - spec: - # Yolo - no pv. - containers: - - name: minio - image: minio/minio:RELEASE.2019-01-31T00-31-19Z - args: - - server - - /data - env: - - name: MINIO_ACCESS_KEY - value: "smth" - - name: MINIO_SECRET_KEY - value: "Need8Chars" - ports: - - containerPort: 9000 \ No newline at end of file diff --git a/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml b/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml deleted file mode 100644 index a4362035e2..0000000000 --- a/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml +++ /dev/null @@ -1,324 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: prometheus -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http-prometheus - name: http-prometheus - selector: - statefulset.kubernetes.io/pod-name: prometheus-0 - type: NodePort ---- -# We want to be able to access each replica. -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: prometheus-1 -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http-prometheus - name: http-prometheus - selector: - statefulset.kubernetes.io/pod-name: prometheus-1 - type: NodePort ---- -# minikube limitation: -# https://github.com/kubernetes/minikube/issues/3351#issuecomment-459898556 -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: sidecar -spec: - ports: - - port: 10901 - protocol: TCP - targetPort: grpc - name: grpc - selector: - statefulset.kubernetes.io/pod-name: prometheus-0 - type: NodePort ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: sidecar-1 -spec: - ports: - - port: 10901 - protocol: TCP - targetPort: grpc - name: grpc - selector: - statefulset.kubernetes.io/pod-name: prometheus-1 - type: NodePort ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: prometheus - labels: - app: prometheus -spec: - serviceName: "prometheus" - replicas: 2 - selector: - matchLabels: - app: prometheus - template: - metadata: - labels: - app: prometheus - # We will use this label to put all StoreAPis - # under the same headless service for - # SRV lookup: thanos-store-api.default.svc - thanos-store-api: "true" - spec: - securityContext: - runAsUser: 1000 - fsGroup: 2000 - runAsNonRoot: true - serviceAccountName: prometheus - containers: - - name: prometheus - image: quay.io/prometheus/prometheus:v2.6.1 - args: - - --config.file=/etc/prometheus-shared/prometheus.yaml - - --storage.tsdb.path=/var/prometheus - - --web.enable-lifecycle - # TODO: Make retention shorter once all old blocks will be uploaded (!) - - --storage.tsdb.retention=2w - # Disable compaction. - - --storage.tsdb.min-block-duration=2h - - --storage.tsdb.max-block-duration=2h - - --web.enable-admin-api - ports: - - name: http-prometheus - containerPort: 9090 - volumeMounts: - - name: config-shared - mountPath: /etc/prometheus-shared - - name: rules - mountPath: /etc/prometheus/rules - - name: prometheus - mountPath: /var/prometheus - - name: thanos - image: improbable/thanos:v0.3.1 - args: - - sidecar - - --log.level=debug - - --tsdb.path=/var/prometheus - - --prometheus.url=http://localhost:9090 - - --cluster.disable - - --reloader.config-file=/etc/prometheus/prometheus.yaml.tmpl - - --reloader.config-envsubst-file=/etc/prometheus-shared/prometheus.yaml - # Enable block uploading. - - | - --objstore.config=type: S3 - config: - bucket: demo-bucket - access_key: smth - secret_key: Need8Chars - endpoint: %%S3_ENDPOINT%% - insecure: true - # New flag for migrating old blocks. - - --shipper.upload-compacted-once - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - ports: - - name: http-sidecar - containerPort: 10902 - - name: grpc - containerPort: 10901 - livenessProbe: - httpGet: - port: 10902 - path: /-/healthy - readinessProbe: - httpGet: - port: 10902 - path: /-/ready - volumeMounts: - - name: prometheus - mountPath: /var/prometheus - - name: config-shared - mountPath: /etc/prometheus-shared - - name: config - mountPath: /etc/prometheus - volumes: - - name: config - configMap: - name: prometheus - - name: rules - configMap: - name: prometheus-rules - - name: config-shared - emptyDir: {} - updateStrategy: - type: RollingUpdate - volumeClaimTemplates: - - metadata: - labels: - app: prometheus - name: prometheus - spec: - storageClassName: prom-manual - accessModes: - - ReadWriteOnce - resources: - requests: - # Normally, probably 15x more (: - storage: 4Gi ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus -data: - prometheus.yaml.tmpl: |- - # Inspired by https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml - global: - scrape_interval: 15s - scrape_timeout: 10s - external_labels: - cluster: %%CLUSTER%% - # Each Prometheus has to have unique labels. - replica: $(POD_NAME) - - alerting: - # We want our alerts to be deduplicated - # from different replicas. - alert_relabel_configs: - - regex: replica - action: labeldrop - - alertmanagers: - - static_configs: - - targets: - - %%ALERTMANAGER_URL%% - - rule_files: - - /etc/prometheus/rules/*rules.yaml - - scrape_configs: - - job_name: kube-apiserver - scheme: https - kubernetes_sd_configs: - - role: endpoints - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - relabel_configs: - - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - regex: default;kubernetes;https - action: keep - - - job_name: kubelet - scheme: https - kubernetes_sd_configs: - - role: node - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - relabel_configs: - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor - metric_relabel_configs: - - source_labels: [pod_name] - regex: ^(.+)$ - target_label: pod - - regex: id - action: labeldrop - - regex: name - action: labeldrop - - regex: pod_name - action: labeldrop - - target_label: cluster - replacement: %%CLUSTER%% - - - job_name: kube-pods - honor_labels: true - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_pod_container_port_name] - regex: ^(http|http-.+|metrics)$ - action: keep - - source_labels: [__meta_kubernetes_pod_label_k8s_app] - target_label: job - - source_labels: [__meta_kubernetes_pod_label_app] - regex: ^(.+)$ - target_label: job - - source_labels: [job, __meta_kubernetes_pod_container_port_name] - regex: ^(.*);http-(.+)$ - target_label: job - - source_labels: [__meta_kubernetes_pod_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - target_label: cluster - replacement: %%CLUSTER%% - metric_relabel_configs: - - source_labels: [pod_name] - regex: ^(.+)$ - target_label: pod - - regex: pod_name - action: labeldrop ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: prometheus - namespace: default -rules: -- apiGroups: [""] - resources: - - nodes - - nodes/proxy - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: - - configmaps - verbs: ["get"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: prometheus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus -subjects: -- kind: ServiceAccount - name: prometheus - namespace: default diff --git a/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml b/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml deleted file mode 100644 index 11b4830360..0000000000 --- a/tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml +++ /dev/null @@ -1,308 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: prometheus -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http-prometheus - name: http-prometheus - selector: - statefulset.kubernetes.io/pod-name: prometheus-0 - type: NodePort ---- -# We want to be able to access each replica. -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: prometheus-1 -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http-prometheus - name: http-prometheus - selector: - statefulset.kubernetes.io/pod-name: prometheus-1 - type: NodePort ---- -# minikube limitation: -# https://github.com/kubernetes/minikube/issues/3351#issuecomment-459898556 -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: sidecar -spec: - ports: - - port: 10901 - protocol: TCP - targetPort: grpc - name: grpc - selector: - statefulset.kubernetes.io/pod-name: prometheus-0 - type: NodePort ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: sidecar-1 -spec: - ports: - - port: 10901 - protocol: TCP - targetPort: grpc - name: grpc - selector: - statefulset.kubernetes.io/pod-name: prometheus-1 - type: NodePort ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: prometheus - labels: - app: prometheus -spec: - serviceName: "prometheus" - replicas: 2 - selector: - matchLabels: - app: prometheus - template: - metadata: - labels: - app: prometheus - # We will use this label to put all StoreAPis - # under the same headless service for - # SRV lookup: thanos-store-api.default.svc - thanos-store-api: "true" - spec: - securityContext: - runAsUser: 1000 - fsGroup: 2000 - runAsNonRoot: true - serviceAccountName: prometheus - containers: - - name: prometheus - image: quay.io/prometheus/prometheus:v2.6.1 - args: - - --config.file=/etc/prometheus-shared/prometheus.yaml - - --storage.tsdb.path=/var/prometheus - - --web.enable-lifecycle - - --storage.tsdb.retention=2w - ports: - - name: http-prometheus - containerPort: 9090 - volumeMounts: - - name: config-shared - mountPath: /etc/prometheus-shared - - name: rules - mountPath: /etc/prometheus/rules - - name: prometheus - mountPath: /var/prometheus - - name: thanos - image: improbable/thanos:v0.3.1 - args: - - sidecar - - --log.level=debug - - --tsdb.path=/var/prometheus - - --prometheus.url=http://localhost:9090 - - --cluster.disable - - --reloader.config-file=/etc/prometheus/prometheus.yaml.tmpl - - --reloader.config-envsubst-file=/etc/prometheus-shared/prometheus.yaml - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - ports: - - name: http-sidecar - containerPort: 10902 - - name: grpc - containerPort: 10901 - livenessProbe: - httpGet: - port: 10902 - path: /-/healthy - readinessProbe: - httpGet: - port: 10902 - path: /-/ready - volumeMounts: - - name: prometheus - mountPath: /var/prometheus - - name: config-shared - mountPath: /etc/prometheus-shared - - name: config - mountPath: /etc/prometheus - volumes: - - name: config - configMap: - name: prometheus - - name: rules - configMap: - name: prometheus-rules - - name: config-shared - emptyDir: {} - updateStrategy: - type: RollingUpdate - volumeClaimTemplates: - - metadata: - labels: - app: prometheus - name: prometheus - spec: - storageClassName: prom-manual - accessModes: - - ReadWriteOnce - resources: - requests: - # Normally, probably 15x more (: - storage: 4Gi ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus -data: - prometheus.yaml.tmpl: |- - # Inspired by https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml - global: - scrape_interval: 15s - scrape_timeout: 10s - external_labels: - cluster: %%CLUSTER%% - # Each Prometheus has to have unique labels. - replica: $(POD_NAME) - - alerting: - # We want our alerts to be deduplicated - # from different replicas. - alert_relabel_configs: - - regex: replica - action: labeldrop - - alertmanagers: - - static_configs: - - targets: - - %%ALERTMANAGER_URL%% - - rule_files: - - /etc/prometheus/rules/*rules.yaml - - scrape_configs: - - job_name: kube-apiserver - scheme: https - kubernetes_sd_configs: - - role: endpoints - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - relabel_configs: - - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - regex: default;kubernetes;https - action: keep - - - job_name: kubelet - scheme: https - kubernetes_sd_configs: - - role: node - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - relabel_configs: - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor - metric_relabel_configs: - - source_labels: [pod_name] - regex: ^(.+)$ - target_label: pod - - regex: id - action: labeldrop - - regex: name - action: labeldrop - - regex: pod_name - action: labeldrop - - target_label: cluster - replacement: %%CLUSTER%% - - - job_name: kube-pods - honor_labels: true - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_pod_container_port_name] - regex: ^(http|http-.+|metrics)$ - action: keep - - source_labels: [__meta_kubernetes_pod_label_k8s_app] - target_label: job - - source_labels: [__meta_kubernetes_pod_label_app] - regex: ^(.+)$ - target_label: job - - source_labels: [job, __meta_kubernetes_pod_container_port_name] - regex: ^(.*);http-(.+)$ - target_label: job - - source_labels: [__meta_kubernetes_pod_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - target_label: cluster - replacement: %%CLUSTER%% - metric_relabel_configs: - - source_labels: [pod_name] - regex: ^(.+)$ - target_label: pod - - regex: pod_name - action: labeldrop ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: prometheus - namespace: default -rules: -- apiGroups: [""] - resources: - - nodes - - nodes/proxy - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: - - configmaps - verbs: ["get"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: prometheus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus -subjects: -- kind: ServiceAccount - name: prometheus - namespace: default diff --git a/tutorials/kubernetes-demo/manifests/prometheus-ha.yaml b/tutorials/kubernetes-demo/manifests/prometheus-ha.yaml deleted file mode 100644 index 5e621ab031..0000000000 --- a/tutorials/kubernetes-demo/manifests/prometheus-ha.yaml +++ /dev/null @@ -1,223 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: prometheus -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http-prometheus - name: http-prometheus - selector: - statefulset.kubernetes.io/pod-name: prometheus-0 - type: NodePort ---- -# We want to be able to access each replica. -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: prometheus-1 -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http-prometheus - name: http-prometheus - selector: - statefulset.kubernetes.io/pod-name: prometheus-1 - type: NodePort ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: prometheus - labels: - app: prometheus -spec: - serviceName: "prometheus" - replicas: 2 - selector: - matchLabels: - app: prometheus - template: - metadata: - labels: - app: prometheus - spec: - securityContext: - runAsUser: 1000 - fsGroup: 2000 - runAsNonRoot: true - serviceAccountName: prometheus - containers: - - name: prometheus - image: quay.io/prometheus/prometheus:v2.6.1 - args: - - --config.file=/etc/prometheus/prometheus.yaml - - --storage.tsdb.path=/var/prometheus - - --web.enable-lifecycle - - --storage.tsdb.retention=2w - ports: - - name: http-prometheus - containerPort: 9090 - volumeMounts: - - name: config - mountPath: /etc/prometheus - - name: rules - mountPath: /etc/prometheus/rules - - name: prometheus - mountPath: /var/prometheus - volumes: - - name: config - configMap: - name: prometheus - - name: rules - configMap: - name: prometheus-rules - volumeClaimTemplates: - - metadata: - labels: - app: prometheus - name: prometheus - spec: - storageClassName: prom-manual - accessModes: - - ReadWriteOnce - resources: - requests: - # Normally, probably 15x more (: - storage: 4Gi ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus -data: - prometheus.yaml: |- - # Inspired by https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml - global: - scrape_interval: 15s - scrape_timeout: 10s - external_labels: - cluster: %%CLUSTER%% - - alerting: - alertmanagers: - - static_configs: - - targets: - - %%ALERTMANAGER_URL%% - - rule_files: - - /etc/prometheus/rules/*rules.yaml - - scrape_configs: - - job_name: kube-apiserver - scheme: https - kubernetes_sd_configs: - - role: endpoints - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - relabel_configs: - - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - regex: default;kubernetes;https - action: keep - - - job_name: kubelet - scheme: https - kubernetes_sd_configs: - - role: node - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - relabel_configs: - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor - metric_relabel_configs: - - source_labels: [pod_name] - regex: ^(.+)$ - target_label: pod - - regex: id - action: labeldrop - - regex: name - action: labeldrop - - regex: pod_name - action: labeldrop - - target_label: cluster - replacement: %%CLUSTER%% - - - job_name: kube-pods - honor_labels: true - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_pod_container_port_name] - regex: ^(http|http-.+|metrics)$ - action: keep - - source_labels: [__meta_kubernetes_pod_label_k8s_app] - target_label: job - - source_labels: [__meta_kubernetes_pod_label_app] - regex: ^(.+)$ - target_label: job - - source_labels: [job, __meta_kubernetes_pod_container_port_name] - regex: ^(.*);http-(.+)$ - target_label: job - - source_labels: [__meta_kubernetes_pod_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - target_label: cluster - replacement: %%CLUSTER%% - metric_relabel_configs: - - source_labels: [pod_name] - regex: ^(.+)$ - target_label: pod - - regex: pod_name - action: labeldrop ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: prometheus - namespace: default -rules: -- apiGroups: [""] - resources: - - nodes - - nodes/proxy - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: - - configmaps - verbs: ["get"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: prometheus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus -subjects: -- kind: ServiceAccount - name: prometheus - namespace: default diff --git a/tutorials/kubernetes-demo/manifests/prometheus-pv-0.yaml b/tutorials/kubernetes-demo/manifests/prometheus-pv-0.yaml deleted file mode 100644 index fc6ad10455..0000000000 --- a/tutorials/kubernetes-demo/manifests/prometheus-pv-0.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Prepare 2 volumes for replicas. It is required only for demo purposes as we artificially generate metrics. -kind: PersistentVolume -apiVersion: v1 -metadata: - name: pv-prometheus-0 - labels: - type: local -spec: - storageClassName: prom-manual - capacity: - storage: 5Gi - accessModes: - - ReadWriteOnce - hostPath: - path: "/data/pv-prometheus-0" \ No newline at end of file diff --git a/tutorials/kubernetes-demo/manifests/prometheus-pv-1.yaml b/tutorials/kubernetes-demo/manifests/prometheus-pv-1.yaml deleted file mode 100644 index 69fce10529..0000000000 --- a/tutorials/kubernetes-demo/manifests/prometheus-pv-1.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: PersistentVolume -apiVersion: v1 -metadata: - name: pv-prometheus-1 - labels: - type: local -spec: - storageClassName: prom-manual - capacity: - storage: 5Gi - accessModes: - - ReadWriteOnce - hostPath: - path: "/data/pv-prometheus-1" \ No newline at end of file diff --git a/tutorials/kubernetes-demo/manifests/prometheus-rules.yaml b/tutorials/kubernetes-demo/manifests/prometheus-rules.yaml deleted file mode 100644 index 2545ad47a1..0000000000 --- a/tutorials/kubernetes-demo/manifests/prometheus-rules.yaml +++ /dev/null @@ -1,949 +0,0 @@ -# Copied from awesome kube-prometheus project (but moved to native configmap): -# https://github.com/coreos/prometheus-operator/blob/master/contrib/kube-prometheus/manifests/prometheus-rules.yaml -# For demo purposes simplification those are adjusted. -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-rules -data: - k8s.rules.yaml: |- - groups: - - name: k8s - rules: - - expr: | - sum(rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container_name!=""}[5m])) by (namespace) - record: namespace:container_cpu_usage_seconds_total:sum_rate - - expr: | - sum by (namespace, pod, container_name) ( - rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container_name!=""}[5m]) - ) - record: namespace_pod_container_name:container_cpu_usage_seconds_total:sum_rate - - expr: | - sum(container_memory_usage_bytes{job="kubelet", image!="", container_name!=""}) by (namespace) - record: namespace:container_memory_usage_bytes:sum - - expr: | - sum by (namespace, label_name) ( - sum(rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container_name!=""}[5m])) by (namespace, pod) - * on (namespace, pod) group_left(label_name) - label_replace(kube_pod_labels{job="kube-state-metrics"}, "pod", "$1", "pod", "(.*)") - ) - record: namespace_name:container_cpu_usage_seconds_total:sum_rate - - expr: | - sum by (namespace, label_name) ( - sum(container_memory_usage_bytes{job="kubelet",image!="", container_name!=""}) by (pod, namespace) - * on (namespace, pod) group_left(label_name) - kube_pod_labels{job="kube-state-metrics"} - ) - record: namespace_name:container_memory_usage_bytes:sum - - expr: | - sum by (namespace, label_name) ( - sum(kube_pod_container_resource_requests_memory_bytes{job="kube-state-metrics"}) by (namespace, pod) - * on (namespace, pod) group_left(label_name) - kube_pod_labels{job="kube-state-metrics"} - ) - record: namespace_name:kube_pod_container_resource_requests_memory_bytes:sum - - expr: | - sum by (namespace, label_name) ( - sum(kube_pod_container_resource_requests_cpu_cores{job="kube-state-metrics"} and on(pod) kube_pod_status_scheduled{condition="true"}) by (namespace, pod) - * on (namespace, pod) group_left(label_name) - kube_pod_labels{job="kube-state-metrics"} - ) - record: namespace_name:kube_pod_container_resource_requests_cpu_cores:sum - kube-scheduler.rules.yaml: |- - groups: - - name: kube-scheduler - rules: - - expr: | - histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.99" - record: cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile - - expr: | - histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.99" - record: cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile - - expr: | - histogram_quantile(0.99, sum(rate(scheduler_binding_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.99" - record: cluster_quantile:scheduler_binding_latency:histogram_quantile - - expr: | - histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.9" - record: cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile - - expr: | - histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.9" - record: cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile - - expr: | - histogram_quantile(0.9, sum(rate(scheduler_binding_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.9" - record: cluster_quantile:scheduler_binding_latency:histogram_quantile - - expr: | - histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.5" - record: cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile - - expr: | - histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.5" - record: cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile - - expr: | - histogram_quantile(0.5, sum(rate(scheduler_binding_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.5" - record: cluster_quantile:scheduler_binding_latency:histogram_quantile - kube-apiserver.rules.yaml: |- - groups: - - name: kube-apiserver - rules: - - expr: | - histogram_quantile(0.99, sum(rate(apiserver_request_latencies_bucket{job="apiserver"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.99" - record: cluster_quantile:apiserver_request_latencies:histogram_quantile - - expr: | - histogram_quantile(0.9, sum(rate(apiserver_request_latencies_bucket{job="apiserver"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.9" - record: cluster_quantile:apiserver_request_latencies:histogram_quantile - - expr: | - histogram_quantile(0.5, sum(rate(apiserver_request_latencies_bucket{job="apiserver"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: "0.5" - record: cluster_quantile:apiserver_request_latencies:histogram_quantile - node.rules.yaml: |- - groups: - - name: node - rules: - - expr: sum(min(kube_pod_info) by (node)) - record: ':kube_pod_info_node_count:' - - expr: | - max(label_replace(kube_pod_info{job="kube-state-metrics"}, "pod", "$1", "pod", "(.*)")) by (node, namespace, pod) - record: 'node_namespace_pod:kube_pod_info:' - - expr: | - count by (node) (sum by (node, cpu) ( - node_cpu_seconds_total{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - )) - record: node:node_num_cpu:sum - - expr: | - 1 - avg(rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m])) - record: :node_cpu_utilisation:avg1m - - expr: | - 1 - avg by (node) ( - rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m]) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info:) - record: node:node_cpu_utilisation:avg1m - - expr: | - sum(node_load1{job="node-exporter"}) - / - sum(node:node_num_cpu:sum) - record: ':node_cpu_saturation_load1:' - - expr: | - sum by (node) ( - node_load1{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - / - node:node_num_cpu:sum - record: 'node:node_cpu_saturation_load1:' - - expr: | - 1 - - sum(node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) - / - sum(node_memory_MemTotal_bytes{job="node-exporter"}) - record: ':node_memory_utilisation:' - - expr: | - sum(node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) - record: :node_memory_MemFreeCachedBuffers_bytes:sum - - expr: | - sum(node_memory_MemTotal_bytes{job="node-exporter"}) - record: :node_memory_MemTotal_bytes:sum - - expr: | - sum by (node) ( - (node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_memory_bytes_available:sum - - expr: | - sum by (node) ( - node_memory_MemTotal_bytes{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_memory_bytes_total:sum - - expr: | - (node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum) - / - scalar(sum(node:node_memory_bytes_total:sum)) - record: node:node_memory_utilisation:ratio - - expr: | - 1e3 * sum( - (rate(node_vmstat_pgpgin{job="node-exporter"}[1m]) - + rate(node_vmstat_pgpgout{job="node-exporter"}[1m])) - ) - record: :node_memory_swap_io_bytes:sum_rate - - expr: | - 1 - - sum by (node) ( - (node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - / - sum by (node) ( - node_memory_MemTotal_bytes{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: 'node:node_memory_utilisation:' - - expr: | - 1 - (node:node_memory_bytes_available:sum / node:node_memory_bytes_total:sum) - record: 'node:node_memory_utilisation_2:' - - expr: | - 1e3 * sum by (node) ( - (rate(node_vmstat_pgpgin{job="node-exporter"}[1m]) - + rate(node_vmstat_pgpgout{job="node-exporter"}[1m])) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_memory_swap_io_bytes:sum_rate - - expr: | - avg(irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+"}[1m])) - record: :node_disk_utilisation:avg_irate - - expr: | - avg by (node) ( - irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+"}[1m]) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_disk_utilisation:avg_irate - - expr: | - avg(irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+"}[1m]) / 1e3) - record: :node_disk_saturation:avg_irate - - expr: | - avg by (node) ( - irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+"}[1m]) / 1e3 - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_disk_saturation:avg_irate - - expr: | - max by (namespace, pod, device) ((node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} - - node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - record: 'node:node_filesystem_usage:' - - expr: | - max by (namespace, pod, device) (node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - record: 'node:node_filesystem_avail:' - - expr: | - sum(irate(node_network_receive_bytes_total{job="node-exporter",device="eth0"}[1m])) + - sum(irate(node_network_transmit_bytes_total{job="node-exporter",device="eth0"}[1m])) - record: :node_net_utilisation:sum_irate - - expr: | - sum by (node) ( - (irate(node_network_receive_bytes_total{job="node-exporter",device="eth0"}[1m]) + - irate(node_network_transmit_bytes_total{job="node-exporter",device="eth0"}[1m])) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_net_utilisation:sum_irate - - expr: | - sum(irate(node_network_receive_drop_total{job="node-exporter",device="eth0"}[1m])) + - sum(irate(node_network_transmit_drop_total{job="node-exporter",device="eth0"}[1m])) - record: :node_net_saturation:sum_irate - - expr: | - sum by (node) ( - (irate(node_network_receive_drop_total{job="node-exporter",device="eth0"}[1m]) + - irate(node_network_transmit_drop_total{job="node-exporter",device="eth0"}[1m])) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_net_saturation:sum_irate - - expr: | - max( - max( - kube_pod_info{job="kube-state-metrics", host_ip!=""} - ) by (node, host_ip) - * on (host_ip) group_right (node) - label_replace( - (max(node_filesystem_files{job="node-exporter", mountpoint="/"}) by (instance)), "host_ip", "$1", "instance", "(.*):.*" - ) - ) by (node) - record: 'node:node_inodes_total:' - - expr: | - max( - max( - kube_pod_info{job="kube-state-metrics", host_ip!=""} - ) by (node, host_ip) - * on (host_ip) group_right (node) - label_replace( - (max(node_filesystem_files_free{job="node-exporter", mountpoint="/"}) by (instance)), "host_ip", "$1", "instance", "(.*):.*" - ) - ) by (node) - record: 'node:node_inodes_free:' - kube-prometheus-node-recording.rules.yaml: |- - groups: - - name: kube-prometheus-node-recording - rules: - - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[3m])) BY - (instance) - record: instance:node_cpu:rate:sum - - expr: sum((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"})) - BY (instance) - record: instance:node_filesystem_usage:sum - - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) - record: instance:node_network_receive_bytes:rate:sum - - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) - record: instance:node_network_transmit_bytes:rate:sum - - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) WITHOUT - (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) - BY (instance, cpu)) BY (instance) - record: instance:node_cpu:ratio - - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) - record: cluster:node_cpu:sum_rate5m - - expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) - BY (instance, cpu)) - record: cluster:node_cpu:ratio - kubernetes-absent.rules.yaml: |- - groups: - - name: kubernetes-absent - rules: - - alert: AlertmanagerDown - annotations: - message: Alertmanager has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-alertmanagerdown - expr: | - absent(up{job="alertmanager-main"} == 1) and up{job="prometheus", cluster="eu1"} > -1 - for: 15m - labels: - severity: critical - - alert: CoreDNSDown - annotations: - message: CoreDNS has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-corednsdown - expr: | - absent(up{job="kube-dns"} == 1) - for: 15m - labels: - severity: critical - - alert: KubeAPIDown - annotations: - message: KubeAPI has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown - expr: | - absent(up{job="kube-apiserver"} == 1) - for: 15m - labels: - severity: critical - - alert: KubeStateMetricsDown - annotations: - message: KubeStateMetrics has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricsdown - expr: | - absent(up{job="kube-state-metrics"} == 1) - for: 15m - labels: - severity: critical - - alert: KubeletDown - annotations: - message: Kubelet has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown - expr: | - absent(up{job="kubelet"} == 1) - for: 15m - labels: - severity: critical - - alert: PrometheusDown - annotations: - message: Prometheus has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusdown - expr: | - absent(up{job="prometheus"} == 1) - for: 15m - labels: - severity: critical - kubernetes-apps.rules.yaml: |- - groups: - - name: kubernetes-apps - rules: - - alert: KubePodCrashLooping - annotations: - message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container - }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping - expr: | - rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0 - for: 1h - labels: - severity: critical - - alert: KubePodNotReady - annotations: - message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready - state for longer than an hour. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready - expr: | - sum by (namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}) > 0 - for: 1h - labels: - severity: critical - - alert: KubeDeploymentGenerationMismatch - annotations: - message: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment - }} does not match, this indicates that the Deployment has failed but has - not been rolled back. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch - expr: | - kube_deployment_status_observed_generation{job="kube-state-metrics"} - != - kube_deployment_metadata_generation{job="kube-state-metrics"} - for: 15m - labels: - severity: critical - - alert: KubeDeploymentReplicasMismatch - annotations: - message: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not - matched the expected number of replicas for longer than an hour. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch - expr: | - kube_deployment_spec_replicas{job="kube-state-metrics"} - != - kube_deployment_status_replicas_available{job="kube-state-metrics"} - for: 1h - labels: - severity: critical - - alert: KubeStatefulSetReplicasMismatch - annotations: - message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has - not matched the expected number of replicas for longer than 15 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch - expr: | - kube_statefulset_status_replicas_ready{job="kube-state-metrics"} - != - kube_statefulset_status_replicas{job="kube-state-metrics"} - for: 15m - labels: - severity: critical - - alert: KubeStatefulSetGenerationMismatch - annotations: - message: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset - }} does not match, this indicates that the StatefulSet has failed but has - not been rolled back. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch - expr: | - kube_statefulset_status_observed_generation{job="kube-state-metrics"} - != - kube_statefulset_metadata_generation{job="kube-state-metrics"} - for: 15m - labels: - severity: critical - - alert: KubeStatefulSetUpdateNotRolledOut - annotations: - message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update - has not been rolled out. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout - expr: | - max without (revision) ( - kube_statefulset_status_current_revision{job="kube-state-metrics"} - unless - kube_statefulset_status_update_revision{job="kube-state-metrics"} - ) - * - ( - kube_statefulset_replicas{job="kube-state-metrics"} - != - kube_statefulset_status_replicas_updated{job="kube-state-metrics"} - ) - for: 15m - labels: - severity: critical - - alert: KubeDaemonSetRolloutStuck - annotations: - message: Only {{ $value }}% of the desired Pods of DaemonSet {{ $labels.namespace - }}/{{ $labels.daemonset }} are scheduled and ready. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck - expr: | - kube_daemonset_status_number_ready{job="kube-state-metrics"} - / - kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} * 100 < 100 - for: 15m - labels: - severity: critical - - alert: KubeDaemonSetNotScheduled - annotations: - message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset - }} are not scheduled.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled - expr: | - kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} - - - kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0 - for: 10m - labels: - severity: warning - - alert: KubeDaemonSetMisScheduled - annotations: - message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset - }} are running where they are not supposed to run.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled - expr: | - kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0 - for: 10m - labels: - severity: warning - - alert: KubeCronJobRunning - annotations: - message: CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more - than 1h to complete. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning - expr: | - time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600 - for: 1h - labels: - severity: warning - - alert: KubeJobCompletion - annotations: - message: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more - than one hour to complete. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion - expr: | - kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0 - for: 1h - labels: - severity: warning - - alert: KubeJobFailed - annotations: - message: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed - expr: | - kube_job_status_failed{job="kube-state-metrics"} > 0 - for: 1h - labels: - severity: warning - kubernetes-resources.rules.yaml: |- - groups: - - name: kubernetes-resources - rules: - - alert: KubeCPUOvercommit - annotations: - message: Cluster has overcommitted CPU resource requests for Pods and cannot - tolerate node failure. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit - expr: | - sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum) - / - sum(node:node_num_cpu:sum) - > - (count(node:node_num_cpu:sum)-1) / count(node:node_num_cpu:sum) - for: 5m - labels: - severity: warning - - alert: KubeMemOvercommit - annotations: - message: Cluster has overcommitted memory resource requests for Pods and cannot - tolerate node failure. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit - expr: | - sum(namespace_name:kube_pod_container_resource_requests_memory_bytes:sum) - / - sum(node_memory_MemTotal_bytes) - > - (count(node:node_num_cpu:sum)-1) - / - count(node:node_num_cpu:sum) - for: 5m - labels: - severity: warning - - alert: KubeCPUOvercommit - annotations: - message: Cluster has overcommitted CPU resource requests for Namespaces. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit - expr: | - sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="requests.cpu"}) - / - sum(node:node_num_cpu:sum) - > 1.5 - for: 5m - labels: - severity: warning - - alert: KubeMemOvercommit - annotations: - message: Cluster has overcommitted memory resource requests for Namespaces. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit - expr: | - sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="requests.memory"}) - / - sum(node_memory_MemTotal_bytes{job="node-exporter"}) - > 1.5 - for: 5m - labels: - severity: warning - - alert: KubeQuotaExceeded - annotations: - message: Namespace {{ $labels.namespace }} is using {{ printf "%0.0f" $value - }}% of its {{ $labels.resource }} quota. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded - expr: | - 100 * kube_resourcequota{job="kube-state-metrics", type="used"} - / ignoring(instance, job, type) - (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) - > 90 - for: 15m - labels: - severity: warning - - alert: CPUThrottlingHigh - annotations: - message: '{{ printf "%0.0f" $value }}% throttling of CPU in namespace {{ $labels.namespace - }} for container {{ $labels.container_name }} in pod {{ $labels.pod - }}.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh - expr: "100 * sum(increase(container_cpu_cfs_throttled_periods_total{container_name!=\"\", - }[5m])) by (container_name, pod, namespace)\n /\nsum(increase(container_cpu_cfs_periods_total{}[5m])) - by (container_name, pod, namespace)\n > 25 \n" - for: 15m - labels: - severity: warning - kubernetes-storage.rules.yaml: |- - groups: - - name: kubernetes-storage - rules: - - alert: KubePersistentVolumeUsageCritical - annotations: - message: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim - }} in Namespace {{ $labels.namespace }} is only {{ printf "%0.2f" $value - }}% free. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical - expr: | - 100 * kubelet_volume_stats_available_bytes{job="kubelet"} - / - kubelet_volume_stats_capacity_bytes{job="kubelet"} - < 3 - for: 1m - labels: - severity: critical - - alert: KubePersistentVolumeFullInFourDays - annotations: - message: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim - }} in Namespace {{ $labels.namespace }} is expected to fill up within four - days. Currently {{ printf "%0.2f" $value }}% is available. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays - expr: | - 100 * ( - kubelet_volume_stats_available_bytes{job="kubelet"} - / - kubelet_volume_stats_capacity_bytes{job="kubelet"} - ) < 15 - and - predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0 - for: 5m - labels: - severity: critical - - alert: KubePersistentVolumeErrors - annotations: - message: The persistent volume {{ $labels.persistentvolume }} has status {{ - $labels.phase }}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors - expr: | - kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 - for: 5m - labels: - severity: critical - kubernetes-system.rules.yaml: |- - groups: - - name: kubernetes-system - rules: - - alert: KubeNodeNotReady - annotations: - message: '{{ $labels.node }} has been unready for more than an hour.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready - expr: | - kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0 - for: 1h - labels: - severity: warning - - alert: KubeVersionMismatch - annotations: - message: There are {{ $value }} different versions of Kubernetes components - running. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch - expr: | - count(count(kubernetes_build_info{job!="kube-dns"}) by (gitVersion)) > 1 - for: 1h - labels: - severity: warning - - alert: KubeClientErrors - annotations: - message: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance - }}' is experiencing {{ printf "%0.0f" $value }}% errors.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors - expr: | - (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job) - / - sum(rate(rest_client_requests_total[5m])) by (instance, job)) - * 100 > 1 - for: 15m - labels: - severity: warning - - alert: KubeClientErrors - annotations: - message: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance - }}' is experiencing {{ printf "%0.0f" $value }} errors / second. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors - expr: | - sum(rate(ksm_scrape_error_total{job="kube-state-metrics"}[5m])) by (instance, job) > 0.1 - for: 15m - labels: - severity: warning - - alert: KubeletTooManyPods - annotations: - message: Kubelet {{ $labels.instance }} is running {{ $value }} Pods, close - to the limit of 110. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods - expr: | - kubelet_running_pod_count{job="kubelet"} > 110 * 0.9 - for: 15m - labels: - severity: warning - - alert: KubeAPILatencyHigh - annotations: - message: The API server has a 99th percentile latency of {{ $value }} seconds - for {{ $labels.verb }} {{ $labels.resource }}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh - expr: | - cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 1 - for: 10m - labels: - severity: warning - - alert: KubeAPILatencyHigh - annotations: - message: The API server has a 99th percentile latency of {{ $value }} seconds - for {{ $labels.verb }} {{ $labels.resource }}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh - expr: | - cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 4 - for: 10m - labels: - severity: critical - - alert: KubeAPIErrorsHigh - annotations: - message: API server is returning errors for {{ $value }}% of requests. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh - expr: | - sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) without(instance, pod) - / - sum(rate(apiserver_request_count{job="apiserver"}[5m])) without(instance, pod) * 100 > 10 - for: 10m - labels: - severity: critical - - alert: KubeAPIErrorsHigh - annotations: - message: API server is returning errors for {{ $value }}% of requests. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh - expr: | - sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) without(instance, pod) - / - sum(rate(apiserver_request_count{job="apiserver"}[5m])) without(instance, pod) * 100 > 5 - for: 10m - labels: - severity: warning - - alert: KubeClientCertificateExpiration - annotations: - message: Kubernetes API certificate is expiring in less than 7 days. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration - expr: | - histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 - labels: - severity: warning - - alert: KubeClientCertificateExpiration - annotations: - message: Kubernetes API certificate is expiring in less than 24 hours. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration - expr: | - histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 - labels: - severity: critical - alertmanager.rules.yaml: |- - groups: - - name: alertmanager - rules: - - alert: AlertmanagerConfigInconsistent - annotations: - message: The configuration of the instances of the Alertmanager cluster `{{$labels.service}}` - are out of sync. - expr: | - count_values("config_hash", alertmanager_config_hash{job="alertmanager-main"}) BY (service) / ON(service) GROUP_LEFT() label_replace(prometheus_operator_spec_replicas{job="prometheus-operator",controller="alertmanager"}, "service", "alertmanager-$1", "name", "(.*)") != 1 - for: 5m - labels: - severity: critical - - alert: AlertmanagerFailedReload - annotations: - message: Reloading Alertmanager's configuration has failed for {{ $labels.namespace - }}/{{ $labels.pod}}. - expr: | - alertmanager_config_last_reload_successful{job="alertmanager-main"} == 0 - for: 10m - labels: - severity: warning - - alert: AlertmanagerMembersInconsistent - annotations: - message: Alertmanager has not found all other members of the cluster. - expr: | - alertmanager_cluster_members{job="alertmanager-main"} - != on (service) GROUP_LEFT() - count by (service) (alertmanager_cluster_members{job="alertmanager-main"}) - for: 5m - labels: - severity: critical - general.rules.yaml: |- - groups: - - name: general - rules: - - alert: TargetDown - annotations: - message: '{{ $value }}% of the {{ $labels.job }} targets are down.' - expr: 100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10 - for: 10m - labels: - severity: warning - - alert: AlertPropagationTest - annotations: - message: This is an AlertPropagationTest meant to ensure that the entire alerting - pipeline is functional. - expr: vector(1) - labels: - severity: none - kube-prometheus-node-alerting.rules.yaml: |- - groups: - - name: kube-prometheus-node-alerting - rules: - - alert: NodeDiskRunningFull - annotations: - message: Device {{ $labels.device }} of node-exporter {{ $labels.namespace - }}/{{ $labels.pod }} will be full within the next 24 hours. - expr: | - (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0) - for: 30m - labels: - severity: warning - - alert: NodeDiskRunningFull - annotations: - message: Device {{ $labels.device }} of node-exporter {{ $labels.namespace - }}/{{ $labels.pod }} will be full within the next 2 hours. - expr: | - (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0) - for: 10m - labels: - severity: critical - prometheus.rules.yaml: |- - groups: - - name: prometheus - rules: - - alert: PrometheusConfigReloadFailed - annotations: - description: Reloading Prometheus' configuration has failed for {{$labels.namespace}}/{{$labels.pod}} - summary: Reloading Prometheus' configuration failed - expr: | - prometheus_config_last_reload_successful{job="prometheus"} == 0 - for: 10m - labels: - severity: warning - - alert: PrometheusNotificationQueueRunningFull - annotations: - description: Prometheus' alert notification queue is running full for {{$labels.namespace}}/{{ - $labels.pod}} - summary: Prometheus' alert notification queue is running full - expr: | - predict_linear(prometheus_notifications_queue_length{job="prometheus"}[5m], 60 * 30) > prometheus_notifications_queue_capacity{job="prometheus"} - for: 10m - labels: - severity: warning - - alert: PrometheusErrorSendingAlerts - annotations: - description: Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ - $labels.pod}} to Alertmanager {{$labels.Alertmanager}} - summary: Errors while sending alert from Prometheus - expr: | - rate(prometheus_notifications_errors_total{job="prometheus"}[5m]) / rate(prometheus_notifications_sent_total{job="prometheus"}[5m]) > 0.01 - for: 10m - labels: - severity: warning - - alert: PrometheusErrorSendingAlerts - annotations: - description: Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ - $labels.pod}} to Alertmanager {{$labels.Alertmanager}} - summary: Errors while sending alerts from Prometheus - expr: | - rate(prometheus_notifications_errors_total{job="prometheus"}[5m]) / rate(prometheus_notifications_sent_total{job="prometheus"}[5m]) > 0.03 - for: 10m - labels: - severity: critical - - alert: PrometheusNotConnectedToAlertmanagers - annotations: - description: Prometheus {{ $labels.namespace }}/{{ $labels.pod}} is not connected - to any Alertmanagers - summary: Prometheus is not connected to any Alertmanagers - expr: | - prometheus_notifications_alertmanagers_discovered{job="prometheus"} < 1 - for: 10m - labels: - severity: warning - - alert: PrometheusTSDBReloadsFailing - annotations: - description: '{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} - reload failures over the last four hours.' - summary: Prometheus has issues reloading data blocks from disk - expr: | - increase(prometheus_tsdb_reloads_failures_total{job="prometheus"}[2h]) > 0 - for: 12h - labels: - severity: warning - - alert: PrometheusTSDBCompactionsFailing - annotations: - description: '{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} - compaction failures over the last four hours.' - summary: Prometheus has issues compacting sample blocks - expr: | - increase(prometheus_tsdb_compactions_failed_total{job="prometheus"}[2h]) > 0 - for: 12h - labels: - severity: warning - - alert: PrometheusTSDBWALCorruptions - annotations: - description: '{{$labels.job}} at {{$labels.instance}} has a corrupted write-ahead - log (WAL).' - summary: Prometheus write-ahead log is corrupted - expr: | - tsdb_wal_corruptions_total{job="prometheus"} > 0 - for: 4h - labels: - severity: warning - - alert: PrometheusNotIngestingSamples - annotations: - description: Prometheus {{ $labels.namespace }}/{{ $labels.pod}} isn't ingesting - samples. - summary: Prometheus isn't ingesting samples - expr: | - rate(prometheus_tsdb_head_samples_appended_total{job="prometheus"}[5m]) <= 0 - for: 10m - labels: - severity: warning - - alert: PrometheusTargetScrapesDuplicate - annotations: - description: '{{$labels.namespace}}/{{$labels.pod}} has many samples rejected - due to duplicate timestamps but different values' - summary: Prometheus has many samples rejected - expr: | - increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus"}[5m]) > 0 - for: 10m - labels: - severity: warning diff --git a/tutorials/kubernetes-demo/manifests/prometheus.yaml b/tutorials/kubernetes-demo/manifests/prometheus.yaml deleted file mode 100644 index 7638ec896a..0000000000 --- a/tutorials/kubernetes-demo/manifests/prometheus.yaml +++ /dev/null @@ -1,206 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus - name: prometheus -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http-prometheus - name: http-prometheus - selector: - statefulset.kubernetes.io/pod-name: prometheus-0 - type: NodePort ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: prometheus - labels: - app: prometheus -spec: - serviceName: "prometheus" - replicas: 1 - selector: - matchLabels: - app: prometheus - template: - metadata: - labels: - app: prometheus - spec: - securityContext: - runAsUser: 1000 - fsGroup: 2000 - runAsNonRoot: true - serviceAccountName: prometheus - containers: - - name: prometheus - image: quay.io/prometheus/prometheus:v2.6.1 - args: - - --config.file=/etc/prometheus/prometheus.yaml - - --storage.tsdb.path=/var/prometheus - - --web.enable-lifecycle - - --storage.tsdb.retention=2w - ports: - - name: http-prometheus - containerPort: 9090 - volumeMounts: - - name: config - mountPath: /etc/prometheus - - name: rules - mountPath: /etc/prometheus/rules - - name: prometheus - mountPath: /var/prometheus - volumes: - - name: config - configMap: - name: prometheus - - name: rules - configMap: - name: prometheus-rules - volumeClaimTemplates: - - metadata: - labels: - app: prometheus - name: prometheus - spec: - storageClassName: prom-manual - accessModes: - - ReadWriteOnce - resources: - requests: - # Normally, probably 15x more (: - storage: 4Gi ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus -data: - prometheus.yaml: |- - # Inspired by https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml - global: - scrape_interval: 15s - scrape_timeout: 10s - external_labels: - cluster: %%CLUSTER%% - - alerting: - alertmanagers: - - static_configs: - - targets: - - %%ALERTMANAGER_URL%% - - rule_files: - - /etc/prometheus/rules/*rules.yaml - - scrape_configs: - - job_name: kube-apiserver - scheme: https - kubernetes_sd_configs: - - role: endpoints - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - relabel_configs: - - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - regex: default;kubernetes;https - action: keep - - - job_name: kubelet - scheme: https - kubernetes_sd_configs: - - role: node - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - relabel_configs: - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor - metric_relabel_configs: - - source_labels: [pod_name] - regex: ^(.+)$ - target_label: pod - - regex: id - action: labeldrop - - regex: name - action: labeldrop - - regex: pod_name - action: labeldrop - - target_label: cluster - replacement: %%CLUSTER%% - - - job_name: kube-pods - honor_labels: true - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_pod_container_port_name] - regex: ^(http|http-.+|metrics)$ - action: keep - - source_labels: [__meta_kubernetes_pod_label_k8s_app] - target_label: job - - source_labels: [__meta_kubernetes_pod_label_app] - regex: ^(.+)$ - target_label: job - - source_labels: [job, __meta_kubernetes_pod_container_port_name] - regex: ^(.*);http-(.+)$ - target_label: job - - source_labels: [__meta_kubernetes_pod_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - target_label: cluster - replacement: %%CLUSTER%% - metric_relabel_configs: - - source_labels: [pod_name] - regex: ^(.+)$ - target_label: pod - - regex: pod_name - action: labeldrop ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: prometheus - namespace: default -rules: -- apiGroups: [""] - resources: - - nodes - - nodes/proxy - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: - - configmaps - verbs: ["get"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: prometheus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus -subjects: -- kind: ServiceAccount - name: prometheus - namespace: default diff --git a/tutorials/kubernetes-demo/manifests/thanos-compactor.yaml b/tutorials/kubernetes-demo/manifests/thanos-compactor.yaml deleted file mode 100644 index 0e0d07fa9f..0000000000 --- a/tutorials/kubernetes-demo/manifests/thanos-compactor.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: thanos-compactor - labels: - app: thanos-compactor -spec: - replicas: 1 - selector: - matchLabels: - app: thanos-compactor - serviceName: thanos-compactor - template: - metadata: - labels: - app: thanos-compactor - spec: - containers: - - name: thanos - image: improbable/thanos:v0.3.1 - args: - - compact - - --log.level=debug - - --data-dir=/data - - | - --objstore.config=type: S3 - config: - bucket: demo-bucket - access_key: smth - secret_key: Need8Chars - endpoint: %%S3_ENDPOINT%% - insecure: true - - --sync-delay=30m - - --wait - ports: - - name: http - containerPort: 10902 - livenessProbe: - httpGet: - port: 10902 - path: /-/healthy - readinessProbe: - httpGet: - port: 10902 - path: /-/ready - resources: - limits: - cpu: "1" - memory: 1Gi - requests: - cpu: "1" - memory: 1Gi diff --git a/tutorials/kubernetes-demo/manifests/thanos-querier-no-us1.yaml b/tutorials/kubernetes-demo/manifests/thanos-querier-no-us1.yaml deleted file mode 100644 index e3d6f4d5e3..0000000000 --- a/tutorials/kubernetes-demo/manifests/thanos-querier-no-us1.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# This allow us to do -# SRV lookup: thanos-store-api.default.svc -apiVersion: v1 -kind: Service -metadata: - name: thanos-store-gateway -spec: - type: ClusterIP - clusterIP: None - ports: - - name: grpc - port: 10901 - targetPort: grpc - selector: - thanos-store-api: "true" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: thanos-querier - labels: - app: thanos-querier -spec: - replicas: 1 - selector: - matchLabels: - app: thanos-querier - template: - metadata: - labels: - app: thanos-querier - spec: - containers: - - name: thanos - image: improbable/thanos:v0.3.1 - args: - - query - - --log.level=debug - - --query.replica-label=replica - - --cluster.disable - # Discover local store APIs using DNS SRV. - - --store=dnssrv+thanos-store-gateway.default.svc - ports: - - name: http - containerPort: 10902 - - name: grpc - containerPort: 10901 - livenessProbe: - httpGet: - path: /-/healthy - port: http ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: thanos-querier - name: thanos-querier -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http - name: http - selector: - app: thanos-querier - type: NodePort diff --git a/tutorials/kubernetes-demo/manifests/thanos-querier.yaml b/tutorials/kubernetes-demo/manifests/thanos-querier.yaml deleted file mode 100644 index e5b50a9717..0000000000 --- a/tutorials/kubernetes-demo/manifests/thanos-querier.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# This allow us to do -# SRV lookup: thanos-store-api.default.svc -apiVersion: v1 -kind: Service -metadata: - name: thanos-store-gateway -spec: - type: ClusterIP - clusterIP: None - ports: - - name: grpc - port: 10901 - targetPort: grpc - selector: - thanos-store-api: "true" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: thanos-querier - labels: - app: thanos-querier -spec: - replicas: 1 - selector: - matchLabels: - app: thanos-querier - template: - metadata: - labels: - app: thanos-querier - spec: - containers: - - name: thanos - image: improbable/thanos:v0.3.1 - args: - - query - - --log.level=debug - - --query.replica-label=replica - - --cluster.disable - # Discover local store APIs using DNS SRV. - - --store=dnssrv+thanos-store-gateway.default.svc - # Get remote store APIs by IP:Port. - - --store=%%SIDECAR_US1_0_URL%% - - --store=%%SIDECAR_US1_1_URL%% - ports: - - name: http - containerPort: 10902 - - name: grpc - containerPort: 10901 - livenessProbe: - httpGet: - port: http - path: /-/healthy - readinessProbe: - httpGet: - port: http - path: /-/ready ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: thanos-querier - name: thanos-querier -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http - name: http - selector: - app: thanos-querier - type: NodePort diff --git a/tutorials/kubernetes-demo/manifests/thanos-ruler.yaml b/tutorials/kubernetes-demo/manifests/thanos-ruler.yaml deleted file mode 100644 index 9adf1589ab..0000000000 --- a/tutorials/kubernetes-demo/manifests/thanos-ruler.yaml +++ /dev/null @@ -1,116 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: thanos-ruler-rules -data: - alert_down_services.rules.yaml: | - groups: - - name: metamonitoring - rules: - - alert: PrometheusReplicaDown - annotations: - message: Prometheus replica in cluster {{$labels.cluster}} has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusdown - expr: | - label_replace( - absent(sum(up{job="prometheus", cluster="eu1", instance=~".*:9090"}) by (job, cluster) == 2), - "cluster", "eu1", "","" - ) - or - label_replace( - absent(sum(up{job="prometheus", cluster="us1", instance=~".*:9090"}) by (job, cluster) == 2), - "cluster", "us1", "","" - ) - for: 15s # for demo purposes - labels: - severity: critical ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - labels: - app: thanos-ruler - name: thanos-ruler -spec: - replicas: 1 - selector: - matchLabels: - app: thanos-ruler - serviceName: thanos-ruler - template: - metadata: - labels: - app: thanos-ruler - thanos-store-api: "true" - spec: - containers: - - name: thanos - image: improbable/thanos:v0.3.1 - args: - - rule - - --log.level=debug - - --data-dir=/data - - --eval-interval=15s - - --cluster.disable - - --rule-file=/etc/thanos-ruler/*.rules.yaml - - --alertmanagers.url=http://%%ALERTMANAGER_URL%% - - --query=thanos-querier.default.svc:9090 - - | - --objstore.config=type: S3 - config: - bucket: demo-bucket - access_key: smth - secret_key: Need8Chars - endpoint: %%S3_ENDPOINT%% - insecure: true - # We don't want to override underlying metric's cluster label. - - --label=ruler_cluster="%%CLUSTER%%" - - --label=replica="$(POD_NAME)" - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - ports: - - name: http - containerPort: 10902 - - name: grpc - containerPort: 10901 - livenessProbe: - httpGet: - port: http - path: /-/healthy - readinessProbe: - httpGet: - port: http - path: /-/ready - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 500m - memory: 500Mi - volumeMounts: - - mountPath: /etc/thanos-ruler - name: config - volumes: - - configMap: - name: thanos-ruler-rules - name: config ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: thanos-ruler - name: thanos-ruler -spec: - ports: - - port: 9090 - protocol: TCP - targetPort: http - name: http - selector: - statefulset.kubernetes.io/pod-name: thanos-ruler-0 - type: NodePort diff --git a/tutorials/kubernetes-demo/manifests/thanos-store-gateway.yaml b/tutorials/kubernetes-demo/manifests/thanos-store-gateway.yaml deleted file mode 100644 index 100d094242..0000000000 --- a/tutorials/kubernetes-demo/manifests/thanos-store-gateway.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: thanos-store-gateway - labels: - app: thanos-store-gateway -spec: - replicas: 1 - selector: - matchLabels: - app: thanos-store-gateway - serviceName: thanos-store-gateway - template: - metadata: - labels: - app: thanos-store-gateway - thanos-store-api: "true" - spec: - containers: - - name: thanos - # TODO(bwplotka): Move to v0.3.2 once available. - image: improbable/thanos:master-2019-02-21-0c730c1 - args: - - store - - --log.level=debug - - --data-dir=/data - - --cluster.disable - - --index-cache-size=500MB - - --chunk-pool-size=500MB - - | - --objstore.config=type: S3 - config: - bucket: demo-bucket - access_key: smth - secret_key: Need8Chars - endpoint: %%S3_ENDPOINT%% - insecure: true - ports: - - name: http - containerPort: 10902 - - name: grpc - containerPort: 10901 - livenessProbe: - httpGet: - port: 10902 - path: /-/healthy - readinessProbe: - httpGet: - port: 10902 - path: /-/ready - resources: - limits: - cpu: "1" - memory: 1Gi - requests: - cpu: "1" - memory: 1Gi - ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: thanos-store-gateway - name: thanos-store-gateway -spec: - ports: - - port: 10901 - protocol: TCP - targetPort: grpc - name: grpc - selector: - app: thanos-store-gateway - type: NodePort diff --git a/tutorials/kubernetes-demo/setup.sh b/tutorials/kubernetes-demo/setup.sh deleted file mode 100755 index 53896a99a1..0000000000 --- a/tutorials/kubernetes-demo/setup.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -set -e - -MINIKUBE_RESTART=${1:true} - -# Prepare setup with: -# -# eu1: -# * Grafana -# * Alertmanager -# * 1 replica Prometheus with 2w metrics data. -# -# us1: -# * 1 replica Prometheus with 2w metrics data. - -if ${MINIKUBE_RESTART}; then - ./cluster-down.sh - ./cluster-up.sh -fi - -kubectl --context=eu1 apply -f manifests/alertmanager.yaml - -sleep 2s - -ALERTMANAGER_URL=$(minikube -p eu1 service alertmanager --format="{{.IP}}:{{.Port}}") -if [[ -z "${ALERTMANAGER_URL}" ]]; then - echo "minikube returns empty result for ALERTMANAGER_URL" - exit 1 -fi - -./apply-pv-gen-metrics.sh eu1 0 336h -kubectl --context=eu1 apply -f manifests/prometheus-rules.yaml -cat manifests/prometheus.yaml | sed "s#%%ALERTMANAGER_URL%%#${ALERTMANAGER_URL}#g" | sed "s#%%CLUSTER%%#eu1#g" | kubectl --context=eu1 apply -f - -kubectl --context=eu1 apply -f manifests/kube-state-metrics.yaml - -./apply-pv-gen-metrics.sh us1 0 336h -kubectl --context=us1 apply -f manifests/prometheus-rules.yaml -cat manifests/prometheus.yaml | sed "s#%%ALERTMANAGER_URL%%#${ALERTMANAGER_URL}#g" | sed "s#%%CLUSTER%%#us1#g" | kubectl --context=us1 apply -f - -kubectl --context=us1 apply -f manifests/kube-state-metrics.yaml - -sleep 1s - -PROM_US1_URL=$(minikube -p us1 service prometheus --url) -echo "PROM_US1_URL=${PROM_US1_URL}" -sed "s#%%PROM_US1_URL%%#${PROM_US1_URL}#g" manifests/grafana-datasources.yaml | kubectl --context=eu1 apply -f - -kubectl apply --context=eu1 -f manifests/grafana.yaml diff --git a/tutorials/kubernetes-demo/slides/globalview-ha.svg b/tutorials/kubernetes-demo/slides/globalview-ha.svg deleted file mode 100644 index 885f60211c..0000000000 --- a/tutorials/kubernetes-demo/slides/globalview-ha.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/tutorials/kubernetes-demo/slides/initial-setup.svg b/tutorials/kubernetes-demo/slides/initial-setup.svg deleted file mode 100644 index c3897c0ff6..0000000000 --- a/tutorials/kubernetes-demo/slides/initial-setup.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/tutorials/kubernetes-demo/slides/unlimited-retention.svg b/tutorials/kubernetes-demo/slides/unlimited-retention.svg deleted file mode 100644 index b13b2a6c46..0000000000 --- a/tutorials/kubernetes-demo/slides/unlimited-retention.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file From a3ab54566ce65741ba37fec5c152d18a80fcfdac Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Thu, 14 Nov 2019 06:31:27 +0100 Subject: [PATCH 049/257] *: Use exponential buckets for histogram metrics (#1545) * Use exponential buckets for compactor histogram metrics Signed-off-by: Kemal Akkoyun * Update buckets Signed-off-by: Kemal Akkoyun * Adjust histogram buckets Signed-off-by: Kemal Akkoyun * Adjust store gate bucket Signed-off-by: Kemal Akkoyun * Adjust http duration buckets Signed-off-by: Kemal Akkoyun --- cmd/thanos/query.go | 4 +--- pkg/compact/compact.go | 16 ++++++---------- pkg/extprom/http/instrument_server.go | 5 +++-- pkg/objstore/objstore.go | 2 +- pkg/server/grpc/grpc.go | 4 +--- pkg/store/bucket.go | 16 ++++++---------- pkg/store/gate.go | 8 +++----- 7 files changed, 21 insertions(+), 34 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 0be1e769b0..3d612d5fdd 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -168,9 +168,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { func storeClientGRPCOpts(logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, secure bool, cert, key, caCert, serverName string) ([]grpc.DialOption, error) { grpcMets := grpc_prometheus.NewClientMetrics() grpcMets.EnableClientHandlingTimeHistogram( - grpc_prometheus.WithHistogramBuckets([]float64{ - 0.001, 0.01, 0.05, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, - }), + grpc_prometheus.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 15)), ) dialOpts := []grpc.DialOption{ // We want to make sure that we can receive huge gRPC messages from storeAPI. diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 01e6f78f89..fbeb9f3b0e 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -81,11 +81,9 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { Help: "Total number of failed sync meta operations.", }) m.syncMetaDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "thanos_compact_sync_meta_duration_seconds", - Help: "Time it took to sync meta files.", - Buckets: []float64{ - 0.25, 0.6, 1, 2, 3.5, 5, 7.5, 10, 15, 30, 60, 100, 200, 500, - }, + Name: "thanos_compact_sync_meta_duration_seconds", + Help: "Time it took to sync meta files.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), }) m.garbageCollectedBlocks = prometheus.NewCounter(prometheus.CounterOpts{ @@ -101,11 +99,9 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { Help: "Total number of failed garbage collection operations.", }) m.garbageCollectionDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "thanos_compact_garbage_collection_duration_seconds", - Help: "Time it took to perform garbage collection iteration.", - Buckets: []float64{ - 0.25, 0.6, 1, 2, 3.5, 5, 7.5, 10, 15, 30, 60, 100, 200, 500, - }, + Name: "thanos_compact_garbage_collection_duration_seconds", + Help: "Time it took to perform garbage collection iteration.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), }) m.compactions = prometheus.NewCounterVec(prometheus.CounterOpts{ diff --git a/pkg/extprom/http/instrument_server.go b/pkg/extprom/http/instrument_server.go index 5a49383b0a..3683592f07 100644 --- a/pkg/extprom/http/instrument_server.go +++ b/pkg/extprom/http/instrument_server.go @@ -39,8 +39,9 @@ func NewInstrumentationMiddleware(reg prometheus.Registerer) InstrumentationMidd ins := defaultInstrumentationMiddleware{ requestDuration: prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Name: "http_request_duration_seconds", - Help: "Tracks the latencies for HTTP requests.", + Name: "http_request_duration_seconds", + Help: "Tracks the latencies for HTTP requests.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 17), }, []string{"code", "handler", "method"}, ), diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index f231d6ef1c..4f8ae11f40 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -204,7 +204,7 @@ func BucketWithMetrics(name string, b Bucket, r prometheus.Registerer) Bucket { Name: "thanos_objstore_bucket_operation_duration_seconds", Help: "Duration of operations against the bucket", ConstLabels: prometheus.Labels{"bucket": name}, - Buckets: []float64{0.005, 0.01, 0.02, 0.04, 0.08, 0.15, 0.3, 0.6, 1, 1.5, 2.5, 5, 10, 20, 30}, + Buckets: prometheus.ExponentialBuckets(0.001, 2, 17), }, []string{"operation"}), lastSuccessfullUploadTime: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_objstore_bucket_last_successful_upload_time", diff --git a/pkg/server/grpc/grpc.go b/pkg/server/grpc/grpc.go index 926b6068aa..725479d797 100644 --- a/pkg/server/grpc/grpc.go +++ b/pkg/server/grpc/grpc.go @@ -43,9 +43,7 @@ func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer met := grpc_prometheus.NewServerMetrics() met.EnableHandlingTimeHistogram( - grpc_prometheus.WithHistogramBuckets([]float64{ - 0.001, 0.01, 0.05, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, - }), + grpc_prometheus.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 15)), ) panicsTotal := prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_grpc_req_panics_recovered_total", diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 3f2c409b3e..a087ba7bd2 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -135,18 +135,14 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { Help: "Number of blocks in a bucket store that were touched to satisfy a query.", }) m.seriesGetAllDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "thanos_bucket_store_series_get_all_duration_seconds", - Help: "Time it takes until all per-block prepares and preloads for a query are finished.", - Buckets: []float64{ - 0.01, 0.05, 0.1, 0.25, 0.6, 1, 2, 3.5, 5, 7.5, 10, 15, 30, 60, - }, + Name: "thanos_bucket_store_series_get_all_duration_seconds", + Help: "Time it takes until all per-block prepares and preloads for a query are finished.", + Buckets: prometheus.ExponentialBuckets(0.01, 2, 15), }) m.seriesMergeDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "thanos_bucket_store_series_merge_duration_seconds", - Help: "Time it takes to merge sub-results from all queried blocks into a single result.", - Buckets: []float64{ - 0.01, 0.05, 0.1, 0.25, 0.6, 1, 2, 3.5, 5, 7.5, 10, 15, 30, 60, - }, + Name: "thanos_bucket_store_series_merge_duration_seconds", + Help: "Time it takes to merge sub-results from all queried blocks into a single result.", + Buckets: prometheus.ExponentialBuckets(0.01, 2, 15), }) m.resultSeriesCount = prometheus.NewSummary(prometheus.SummaryOpts{ Name: "thanos_bucket_store_series_result_series", diff --git a/pkg/store/gate.go b/pkg/store/gate.go index cdb9ea3712..1a8fe69108 100644 --- a/pkg/store/gate.go +++ b/pkg/store/gate.go @@ -24,11 +24,9 @@ func NewGate(maxConcurrent int, reg prometheus.Registerer) *Gate { Help: "Number of queries that are currently in flight.", }), gateTiming: prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "gate_duration_seconds", - Help: "How many seconds it took for queries to wait at the gate.", - Buckets: []float64{ - 0.01, 0.05, 0.1, 0.25, 0.6, 1, 2, 3.5, 5, 10, - }, + Name: "gate_duration_seconds", + Help: "How many seconds it took for queries to wait at the gate.", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 15), }), } From 5daf78710d4cf5b852dd47746a0da15fd1bab233 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 15 Nov 2019 13:20:05 +0000 Subject: [PATCH 050/257] Updated all deps. (#1748) Signed-off-by: Bartek Plotka --- CHANGELOG.md | 1 + go.mod | 97 +++++-- go.sum | 305 +++++++++++++++------- pkg/compact/downsample/downsample_test.go | 11 +- 4 files changed, 297 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f5463f2c4..7301a46339 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1666](https://github.com/thanos-io/thanos/pull/1666) `thanos_compact_group_compactions_total` now counts block compactions, so operations that resulted in a compacted block. The old behaviour is now exposed by new metric: `thanos_compact_group_compaction_runs_started_total` and `thanos_compact_group_compaction_runs_completed_total` which counts compaction runs overall. +- [#1748](https://github.com/thanos-io/thanos/pull/1748) Updated all dependencies. ## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14 diff --git a/go.mod b/go.mod index 9ab5c65838..de1d4ed9b9 100644 --- a/go.mod +++ b/go.mod @@ -1,66 +1,117 @@ module github.com/thanos-io/thanos require ( - cloud.google.com/go v0.44.1 - github.com/Azure/azure-storage-blob-go v0.7.0 + cloud.google.com/go v0.48.0 + cloud.google.com/go/bigquery v1.3.0 // indirect + cloud.google.com/go/storage v1.3.0 + github.com/Azure/azure-pipeline-go v0.2.2 // indirect + github.com/Azure/azure-sdk-for-go v36.1.0+incompatible // indirect + github.com/Azure/azure-storage-blob-go v0.8.0 + github.com/Azure/go-autorest/autorest v0.9.3-0.20191028180845-3492b2aff503 // indirect + github.com/Azure/go-autorest/autorest/adal v0.8.1-0.20191028180845-3492b2aff503 // indirect + github.com/Azure/go-autorest/autorest/to v0.3.1-0.20191028180845-3492b2aff503 // indirect + github.com/Azure/go-autorest/autorest/validation v0.2.1-0.20191028180845-3492b2aff503 // indirect github.com/NYTimes/gziphandler v1.1.1 - github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible - github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 + github.com/OneOfOne/xxhash v1.2.6 // indirect + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect + github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible + github.com/armon/go-metrics v0.3.0 + github.com/aws/aws-sdk-go v1.25.35 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/cespare/xxhash v1.1.0 + github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/fatih/structtag v1.0.0 + github.com/elastic/go-sysinfo v1.1.1 // indirect + github.com/elastic/go-windows v1.0.1 // indirect + github.com/evanphx/json-patch v4.5.0+incompatible // indirect + github.com/fatih/structtag v1.1.0 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 github.com/go-kit/kit v0.9.0 - github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 + github.com/gogo/protobuf v1.3.1 + github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect github.com/golang/snappy v0.0.1 github.com/googleapis/gax-go v2.0.2+incompatible - github.com/gophercloud/gophercloud v0.3.0 + github.com/googleapis/gnostic v0.3.1 // indirect + github.com/gophercloud/gophercloud v0.6.0 + github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/grpc-gateway v1.12.1 // indirect + github.com/hashicorp/consul/api v1.3.0 // indirect + github.com/hashicorp/go-immutable-radix v1.1.0 // indirect + github.com/hashicorp/go-msgpack v0.5.5 // indirect + github.com/hashicorp/go-rootcerts v1.0.1 // indirect github.com/hashicorp/golang-lru v0.5.3 + github.com/hashicorp/memberlist v0.1.5 // indirect + github.com/hashicorp/serf v0.8.5 // indirect + github.com/json-iterator/go v1.1.8 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/leanovate/gopter v0.2.4 github.com/lovoo/gcloud-opentracing v0.3.0 - github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect; Pinned for FreeBSD support. - github.com/mattn/go-runewidth v0.0.4 // indirect + github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe // indirect + github.com/mattn/go-runewidth v0.0.6 // indirect github.com/miekg/dns v1.1.22 - github.com/minio/minio-go/v6 v6.0.39 - github.com/mozillazg/go-cos v0.12.0 + github.com/minio/minio-go/v6 v6.0.41 + github.com/mozillazg/go-cos v0.13.0 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/oklog/run v1.0.0 github.com/oklog/ulid v1.3.1 - github.com/olekukonko/tablewriter v0.0.1 + github.com/olekukonko/tablewriter v0.0.2 + github.com/onsi/ginkgo v1.10.3 // indirect + github.com/onsi/gomega v1.7.1 // indirect github.com/opentracing/basictracer-go v1.0.0 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.2.1 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 github.com/prometheus/common v0.7.0 - github.com/prometheus/prometheus v1.8.2-0.20190913102521-8ab628b35467 // v1.8.2 is misleading as Prometheus does not have v2 module. - github.com/uber-go/atomic v1.4.0 // indirect - github.com/uber/jaeger-client-go v2.16.0+incompatible + github.com/prometheus/procfs v0.0.6 // indirect + github.com/prometheus/prometheus v1.8.2-0.20191114185310-85cae37107c2 // Prometheus master v2.14.0 + github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect + github.com/satori/go.uuid v1.2.0 // indirect + github.com/smartystreets/assertions v1.0.1 // indirect + github.com/smartystreets/goconvey v1.6.4 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/uber/jaeger-client-go v2.20.1+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible go.elastic.co/apm v1.5.0 go.elastic.co/apm/module/apmot v1.5.0 + go.opencensus.io v0.22.2 // indirect + go.uber.org/atomic v1.5.0 // indirect go.uber.org/automaxprocs v1.2.0 - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect - golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 // indirect + golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 - golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 // indirect golang.org/x/text v0.3.2 - google.golang.org/api v0.11.0 - google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 - google.golang.org/grpc v1.22.1 + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + golang.org/x/tools v0.0.0-20191114222411-4191b8cbba09 // indirect + google.golang.org/api v0.13.0 + google.golang.org/appengine v1.6.5 // indirect + google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de + google.golang.org/grpc v1.25.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/fsnotify.v1 v1.4.7 - gopkg.in/yaml.v2 v2.2.2 + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.51.0 // indirect + gopkg.in/yaml.v2 v2.2.5 + k8s.io/api v0.0.0-20191115095533-47f6de673b26 // indirect + k8s.io/apimachinery v0.0.0-20191115015347-3c7067801da2 // indirect + k8s.io/klog v1.0.0 // indirect + k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a // indirect + k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6 // indirect ) // We want to replace the client-go version with a specific commit hash, // so that we don't get errors about being incompatible with the Go proxies. // See https://github.com/thanos-io/thanos/issues/1415 replace ( + // Mitigation for: https://github.com/Azure/go-autorest/issues/414 + github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.3.0+incompatible golang.org/x/sys => golang.org/x/sys v0.0.0-20190412213103-97732733099d // v0.0.0-20190425145619-16072639606e (multiple-value "golang.org/x/sys/windows".GetCurrentProcess() in single-value context) Required to build properly on windows for github.com/elastic/go-sysinfo. k8s.io/api => k8s.io/api v0.0.0-20190620084959-7cf5895f2711 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190620085554-14e95df34f1f diff --git a/go.sum b/go.sum index 2ec5b5a3cd..f049dfe801 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,74 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1 h1:7gXaI3V/b4DRaK++rTqhRajcT7z8gtP0qKMZTXqlySM= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.48.0 h1:6ZHYIRlohUdU4LrLHbTsReY1eYy/MoZW1FsEyBuMXsk= +cloud.google.com/go v0.48.0/go.mod h1:gGOnoa/XMQYHAscREBlbdHduGchEaP9N0//OXdrPI/M= +cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.3.0 h1:2Ze/3nQD5F+HfL0xOPM2EeawDWs+NPRtzgcre+17iZU= +cloud.google.com/go/storage v1.3.0/go.mod h1:9IAwXhoyBJ7z9LcAwkj0/7NnPzYaPeZxxVp3zm+5IqA= contrib.go.opencensus.io/exporter/ocagent v0.6.0 h1:Z1n6UAyr0QwM284yUuh5Zd8JlvxUGAhFZcgMJkMPrGM= contrib.go.opencensus.io/exporter/ocagent v0.6.0/go.mod h1:zmKjrJcdo0aYcVS7bmEeSEBLPA9YJp5bjrofdU3pIXs= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-sdk-for-go v23.2.0+incompatible h1:bch1RS060vGpHpY3zvQDV4rOiRw25J1zmR/B9a76aSA= github.com/Azure/azure-sdk-for-go v23.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-storage-blob-go v0.7.0 h1:MuueVOYkufCxJw5YZzF842DY2MBsp+hLuh2apKY0mck= -github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= -github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v11.2.8+incompatible h1:Q2feRPMlcfVcqz3pF87PJzkm5lZrL+x6BDtzhODzNJM= -github.com/Azure/go-autorest v11.2.8+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/azure-sdk-for-go v36.1.0+incompatible h1:smHlbChr/JDmsyUqELZXLs0YIgpXecIGdUibuc2983s= +github.com/Azure/azure-sdk-for-go v36.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= +github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/Azure/go-autorest v12.3.0+incompatible h1:iw0EvmwwEhv8JzEFfbKNJjnrHJqiH5NlKqhdYiKXRUQ= +github.com/Azure/go-autorest v12.3.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.3-0.20191028180845-3492b2aff503 h1:uUhdsDMg2GbFLF5GfQPtLMWd5vdDZSfqvqQp3waafxQ= +github.com/Azure/go-autorest/autorest v0.9.3-0.20191028180845-3492b2aff503/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.1-0.20191028180845-3492b2aff503 h1:Hxqlh1uAA8aGpa1dFhDNhll7U/rkWtG8ZItFvRMr7l0= +github.com/Azure/go-autorest/autorest/adal v0.8.1-0.20191028180845-3492b2aff503/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/to v0.3.1-0.20191028180845-3492b2aff503 h1:2McfZNaDqGPjv2pddK547PENIk4HV+NT7gvqRq4L0us= +github.com/Azure/go-autorest/autorest/to v0.3.1-0.20191028180845-3492b2aff503/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= +github.com/Azure/go-autorest/autorest/validation v0.2.1-0.20191028180845-3492b2aff503 h1:RBrGlrkPWapMcLp1M6ywCqyYKOAT5ERI6lYFvGKOThE= +github.com/Azure/go-autorest/autorest/validation v0.2.1-0.20191028180845-3492b2aff503/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= -github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/OneOfOne/xxhash v1.2.6 h1:U68crOE3y3MPttCMQGywZOLrTeF5HHJ3/vDBCJn9/bA= +github.com/OneOfOne/xxhash v1.2.6/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -35,13 +78,16 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZq github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible h1:/MzpJOMHn/uBtd1dkS7Q9PF2ZjT6xTQMXSvv1e6ydXc= -github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible h1:EaK5256H3ELiyaq5O/Zwd6fnghD6DqmZDQmmzzJklUU= +github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/armon/go-metrics v0.3.0 h1:B7AQgHi8QSEi4uHu7Sbsga+IJDU+CENgjxoo81vDUqU= +github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -49,6 +95,8 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.23.12 h1:2UnxgNO6Y5J1OrkXS8XNp0UatDxD1bWHiDT62RDPggI= github.com/aws/aws-sdk-go v1.23.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.25.35 h1:fe2tJnqty/v/50pyngKdNk/NP8PFphYDA1Z7N3EiiiE= +github.com/aws/aws-sdk-go v1.25.35/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= @@ -66,6 +114,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -76,24 +126,33 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda h1:NyywMz59neOoVRFDz+ccfKWxn784fiHMDnZSy6T+JXY= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/go-sysinfo v1.0.1 h1:lzGPX2sIXaETeMXitXL2XZU8K4B7k7JBhIKWxdOdUt8= github.com/elastic/go-sysinfo v1.0.1/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= +github.com/elastic/go-sysinfo v1.1.1 h1:ZVlaLDyhVkDfjwPGU55CQRCRolNpc7P0BbyhhQZQmMI= +github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= +github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550 h1:mV9jbLoSW/8m4VK16ZkHTozJa8sesK5u5kTMFysTYac= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/structtag v1.0.0 h1:pTHj65+u3RKWYPSGaU290FpI/dXxTaHdVwVwbcPKmEc= -github.com/fatih/structtag v1.0.0/go.mod h1:IKitwq45uXL/yqi5mYghiD3w9H6eTOvI9vnk8tXMphA= +github.com/fatih/structtag v1.1.0 h1:6j4mUV/ES2duvnAzKMFkN6/A5mCaNYPD3xfbAkLLOF8= +github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -101,6 +160,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= @@ -111,47 +171,25 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.17.2/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.17.2/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.18.0/go.mod h1:uI6pHuxWYTy94zZxgcwJkUWa9wbIlhteGfloI10GD4U= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.3/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.17.2/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.4/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -160,11 +198,16 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 h1:X+zN6RZXsvnrSJaAIQhZezPfAfvsqihKKR8oiLHid34= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE= +github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -182,6 +225,7 @@ github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -196,6 +240,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -204,26 +249,37 @@ github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= -github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= +github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= github.com/gophercloud/gophercloud v0.3.0 h1:6sjpKIpVwRIIwmcEGp+WwNovNsem+c+2vm6oxshRpL8= github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gophercloud/gophercloud v0.6.0 h1:Xb2lcqZtml1XjgYZxbeayEemq7ASbeTp09m36gQFpEU= +github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de h1:F7WD09S8QB4LrkEpka0dFPLSotH11HRpCsLIbIcJ7sU= +github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 h1:uGoIog/wiQHI9GAxXO5TJbT0wWKH3O9HhOJW1F9c3fY= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340/go.mod h1:3bDW6wMZJB7tiONtC/1Xpicra6Wp5GgbTbQWCbI5fkc= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.12.1 h1:zCy2xE9ablevUOrUZc3Dl72Dt+ya2FNAvC2yLYMHzi4= +github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.3.0 h1:HXNYlRkkM/t+Y/Yhxtwcy02dlYwIaoxzvxPnS+cqy78= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0 h1:UOxjlb4xVNF93jak1mzzoBatyFju9nrkxpVwIp/QqxQ= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -233,6 +289,7 @@ github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxB github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -260,12 +317,12 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs= -github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM= +github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.8.3 h1:MWYcmct5EtKz0efYooPcL0yNkem+7kWxqXDi/UIh+8k= -github.com/hashicorp/serf v0.8.3/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k= +github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc= +github.com/hashicorp/serf v0.8.5/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -276,18 +333,25 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5i github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= -github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= -github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -297,7 +361,6 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= @@ -306,17 +369,17 @@ github.com/leanovate/gopter v0.2.4/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkO github.com/lovoo/gcloud-opentracing v0.3.0 h1:nAeKG70rIsog0TelcEtt6KU0Y1s5qXtsDLnHp0urPLU= github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw= -github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe h1:YioO2TiJyAHWHyCRQCP8jk5IzTqmsbGc5qQPIhHo6xs= +github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.6 h1:V2iyH+aX9C5fsYCpK60U8BYIvmhqxuOL3JZcqc1NB7k= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -324,11 +387,12 @@ github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc= github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/minio/minio-go/v6 v6.0.39 h1:9qmKCTBpQpMdGlDAbs3mbb4mmL45/lwRUvHL1VLhYUk= -github.com/minio/minio-go/v6 v6.0.39/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/minio-go/v6 v6.0.41 h1:ybV6itOYLsjCpp4+QjE4gokic/xhJU7D7wRzxqPHTio= +github.com/minio/minio-go/v6 v6.0.41/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -346,8 +410,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mozillazg/go-cos v0.12.0 h1:b9hUd5HjrDe10BUfkyiLYI1+z4M2kAgKasktszx9pO4= -github.com/mozillazg/go-cos v0.12.0/go.mod h1:Zp6DvvXn0RUOXGJ2chmWt2bLEqRAnJnS3DnAZsJsoaE= +github.com/mozillazg/go-cos v0.13.0 h1:RylOpEESdWMLb13bl0ADhko12uMN3JmHqqwFu4OYGBY= +github.com/mozillazg/go-cos v0.13.0/go.mod h1:Zp6DvvXn0RUOXGJ2chmWt2bLEqRAnJnS3DnAZsJsoaE= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= @@ -362,10 +426,16 @@ github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.2 h1:sq53g+DWf0J6/ceFUHpQ0nAEb6WgM++fq16MZ91cS6o= +github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3 h1:EooPXg51Tn+xmWPXJUGCnJhJSpeuMlBmfJVcqIRmmv8= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 h1:QsgXACQhd9QJhEmRumbsMQQvBtmdS0mafoVEBplWXEg= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= @@ -389,8 +459,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.2.0 h1:g4yo/h/me4ZL9o0SVHNRdS2jn5SY8GDmMgkhQ8Mz70s= +github.com/prometheus/client_golang v1.2.0/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -401,8 +471,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -414,19 +482,27 @@ github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURm github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.6 h1:0qbH+Yqu/cj1ViVLvEWCP6qMQ4efWUj6bQqOEA0V0U4= +github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= -github.com/prometheus/prometheus v1.8.2-0.20190913102521-8ab628b35467 h1:B9IMa7s163/ZDSduepHHfOZZHSKdSbgo/bFY5c+FMAs= -github.com/prometheus/prometheus v1.8.2-0.20190913102521-8ab628b35467/go.mod h1:aojjoH+vNHyJUTJoW15HoQWMKXxNhQylU6/G261nqxQ= +github.com/prometheus/prometheus v1.8.2-0.20191114185310-85cae37107c2 h1:33wX5JpFUQiXXSWO/g8a9MwYj8VzEno7s5bpJpVEtN8= +github.com/prometheus/prometheus v1.8.2-0.20191114185310-85cae37107c2/go.mod h1:PVTKYlgELGIDbIKIyWRzD4WKjnavPynGOFLSuDpvOwU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190810000440-0ceca61e4d75 h1:cA+Ubq9qEVIQhIWvP2kNuSZ2CmnfBJFSRq+kO1pu2cc= github.com/samuel/go-zookeeper v0.0.0-20190810000440-0ceca61e4d75/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef h1:RoeI7K0oZIcUirMHsFpQjTVDrl1ouNh8T7v3eNsUxL0= github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= @@ -437,8 +513,12 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= +github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -447,6 +527,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -457,10 +539,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= -github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= -github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= -github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.20.1+incompatible h1:HgqpYBng0n7tLJIlyT4kPCIv5XgCsF+kai1NnnrJzEU= +github.com/uber/jaeger-client-go v2.20.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= @@ -473,12 +553,15 @@ go.elastic.co/apm/module/apmot v1.5.0/go.mod h1:d2KYwhJParTpyw2WnTNy8geNlHKKFX+4 go.elastic.co/fastjson v1.0.0 h1:ooXV/ABvf+tBul26jcVViPT3sBir0PvXgibYB1IQQzg= go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.0.4/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/automaxprocs v1.2.0 h1:+RUihKM+nmYUoB9w0D0Ov5TJ2PpFO2FgenTxMJiZBZA= go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -487,29 +570,37 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= +golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -522,7 +613,6 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -536,8 +626,9 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smto golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 h1:p9xBe/w/OzkeYVKm234g55gMdD1nSIooTir5kV11kfA= -golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -551,6 +642,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -564,8 +657,9 @@ golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -579,25 +673,38 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190813034749-528a2984e271/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18 h1:xFbv3LvlvQAmbNJFCBKRv1Ccvnh9FVsW0FX2kTWWowE= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190918214516-5a1a30219888/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191111182352-50fa39b762bc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191114222411-4191b8cbba09 h1:f3ZhcxGnJxVhcsQgKPvfAg2pjdeWBBgDuO5XfmGnoU0= +golang.org/x/tools v0.0.0-20191114222411-4191b8cbba09/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0 h1:VGGbLNyPF7dvYHhcUGYBBGCRDDK0RRJAI6KCvo0CL+E= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0= -google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -609,27 +716,41 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 h1:iKtrH9Y8mcbADOP google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de h1:dFEMUWudT9iV1JMk6i6NwbfIw2V/2VDFyDYCZFypRxE= +google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.22.1 h1:/7cs52RnTJmD43s3uxzlq2U7nqVTd/37viQwMrMNlOM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/fsnotify/fsnotify.v1 v1.4.7 h1:XNNYLJHt73EyYiCZi6+xjupS9CpvmiDgjPTAjrBlQbo= gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= +gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -637,12 +758,17 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= k8s.io/api v0.0.0-20190620084959-7cf5895f2711 h1:BblVYz/wE5WtBsD/Gvu54KyBUTJMflolzc5I2DTvh50= @@ -655,9 +781,10 @@ k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= -k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a h1:uy5HAgt4Ha5rEMbhZA+aM1j2cq5LmR6LQ71EYC2sVH4= -k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6 h1:p0Ai3qVtkbCG/Af26dBmU0E1W58NID3hSSh7cMyylpM= +k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/compact/downsample/downsample_test.go b/pkg/compact/downsample/downsample_test.go index 3c0fce8851..34e4eaa26f 100644 --- a/pkg/compact/downsample/downsample_test.go +++ b/pkg/compact/downsample/downsample_test.go @@ -18,6 +18,7 @@ import ( "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/index" "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/tsdb/tombstones" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/testutil" @@ -615,7 +616,7 @@ func (b *memBlock) Chunks() (tsdb.ChunkReader, error) { return b, nil } -func (b *memBlock) Tombstones() (tsdb.TombstoneReader, error) { +func (b *memBlock) Tombstones() (tombstones.Reader, error) { return emptyTombstoneReader{}, nil } @@ -625,7 +626,7 @@ func (b *memBlock) Close() error { type emptyTombstoneReader struct{} -func (emptyTombstoneReader) Get(ref uint64) (tsdb.Intervals, error) { return nil, nil } -func (emptyTombstoneReader) Iter(func(uint64, tsdb.Intervals) error) error { return nil } -func (emptyTombstoneReader) Total() uint64 { return 0 } -func (emptyTombstoneReader) Close() error { return nil } +func (emptyTombstoneReader) Get(ref uint64) (tombstones.Intervals, error) { return nil, nil } +func (emptyTombstoneReader) Iter(func(uint64, tombstones.Intervals) error) error { return nil } +func (emptyTombstoneReader) Total() uint64 { return 0 } +func (emptyTombstoneReader) Close() error { return nil } From ee727313ea4934873cd6c58e03b52d06929adbc7 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 18 Nov 2019 17:38:16 +0000 Subject: [PATCH 051/257] query: Fixed labelset view for multi labels scenario. (#1751) Signed-off-by: Bartek Plotka --- pkg/ui/bindata.go | 160 +++++++++++++++++------------------ pkg/ui/templates/stores.html | 24 +++--- 2 files changed, 94 insertions(+), 90 deletions(-) diff --git a/pkg/ui/bindata.go b/pkg/ui/bindata.go index 1e44bf2508..c189484a7f 100644 --- a/pkg/ui/bindata.go +++ b/pkg/ui/bindata.go @@ -115,32 +115,32 @@ type bindataFileInfo struct { modTime time.Time } -// Name return file name. +// Name return file name func (fi bindataFileInfo) Name() string { return fi.name } -// Size return file size. +// Size return file size func (fi bindataFileInfo) Size() int64 { return fi.size } -// Mode return file mode. +// Mode return file mode func (fi bindataFileInfo) Mode() os.FileMode { return fi.mode } -// Mode return file modify time. +// Mode return file modify time func (fi bindataFileInfo) ModTime() time.Time { return fi.modTime } -// IsDir return file whether a directory. +// IsDir return file whether a directory func (fi bindataFileInfo) IsDir() bool { return fi.mode&os.ModeDir != 0 } -// Sys return file is sys mode. +// Sys return file is sys mode func (fi bindataFileInfo) Sys() interface{} { return nil } @@ -160,7 +160,7 @@ func pkgUiTemplates_baseHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/_base.html", size: 1478, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/_base.html", size: 1478, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -180,7 +180,7 @@ func pkgUiTemplatesAlertsHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 2696, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 2696, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -200,7 +200,7 @@ func pkgUiTemplatesBucketHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket.html", size: 1281, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket.html", size: 1281, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -220,7 +220,7 @@ func pkgUiTemplatesBucket_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 785, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 785, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -240,7 +240,7 @@ func pkgUiTemplatesGraphHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/graph.html", size: 2298, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/graph.html", size: 2298, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -260,7 +260,7 @@ func pkgUiTemplatesQuery_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1364, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1364, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -280,7 +280,7 @@ func pkgUiTemplatesRule_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 961, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 961, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -300,7 +300,7 @@ func pkgUiTemplatesRulesHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rules.html", size: 1944, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rules.html", size: 1944, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -320,12 +320,12 @@ func pkgUiTemplatesStatusHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/status.html", size: 1272, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/status.html", size: 1272, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _pkgUiTemplatesStoresHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x55\x51\x6b\xdb\x30\x10\x7e\xcf\xaf\x38\x4c\x5f\x13\x43\x5f\x06\xc3\xc9\x18\xa3\xb0\x87\xb6\x0c\xd2\xf5\x75\x28\xd6\x25\x16\x55\x24\xa3\x3b\xb7\x09\x42\xff\x7d\xc8\xb1\x13\xbb\x76\xb2\x64\x2f\x02\xdd\x7d\xa7\xfb\x74\xf7\xe9\xe4\xbd\xc4\xb5\x32\x08\x49\x81\x42\x26\x21\x4c\x32\xad\xcc\x1b\xf0\xbe\xc4\x79\xc2\xb8\xe3\x34\x27\x4a\xc0\xa1\x9e\x27\xc4\x7b\x8d\x54\x20\x72\x02\x85\xc3\xf5\x3c\xf1\x1e\x4a\xc1\xc5\x2f\x87\x6b\xb5\x83\x10\x52\x62\xc1\x2a\x8f\x31\xa9\xab\x34\xd2\x2c\x27\xfa\xf6\x3e\xf7\x1e\x56\x95\xd2\xf2\x15\x1d\x29\x6b\x20\x84\x64\x31\xf1\x1e\x8d\x0c\x61\x32\x39\x91\xc8\xad\x61\x34\x5c\xf3\x90\xea\x1d\x72\x2d\x88\xe6\xb5\x59\x28\x83\x6e\xba\xd6\x95\x92\xc9\x62\x02\x00\xe0\xbd\x13\x66\x83\x70\x47\x6c\x1d\xbe\xec\x4b\x84\xaf\x73\x98\x2d\x6d\xe5\x72\xa4\x10\x1a\x90\x5a\x77\x10\x8d\x35\x2b\xee\x17\xde\xb3\x62\xdd\x0d\x9f\x2d\xd9\x29\xb3\x09\x21\x4b\x8b\xfb\x36\x07\x6a\xea\x46\xfd\x36\x6f\xc6\x7e\x18\x88\xf8\x1e\xac\xbe\x4a\x8d\x62\xb1\xd2\xd8\x52\x3f\x6c\xea\x75\xba\xb2\x4e\xa2\xc3\x96\xff\x01\x1c\xeb\xde\xdd\xbb\xd3\xa6\x01\x2c\x1e\x8c\x2c\xad\x32\x9c\xa5\x5c\x0c\xbd\x4b\x16\x5c\xd1\xb8\xef\xbb\x31\xb6\x32\x39\x4a\x78\x14\x2b\xd4\x4b\xe4\x33\xc0\x27\x65\xe0\x45\x6d\xf1\x8c\x57\xec\x2e\x78\x1f\x05\x31\xfc\x44\xa1\xb9\x80\x1f\x05\xe6\x6f\x17\x60\x4f\x48\x24\x36\x9f\x0e\xca\xd2\xee\xad\xa3\xef\x53\x4d\x56\x56\xee\x4f\xfb\x7e\xdf\x63\xcf\x95\x91\xb8\x83\xbb\xd9\x32\x1a\x68\xd8\xee\x33\x95\x95\x0b\xef\x0f\xd8\xd9\xb3\xd8\x62\xec\x3b\xcb\x01\xa8\xed\x64\x94\x36\x26\x7d\x37\xb4\x0a\x33\x96\x9b\xb4\xb3\x78\xcf\x07\xe7\xac\xeb\x24\x3f\x1e\x47\xa5\x30\xed\x81\x42\xa3\x63\xa8\xd7\x29\x55\x79\x8e\x44\x50\x27\xf9\xa3\x8c\x54\xb9\x60\xeb\x20\xbe\xc0\x69\x55\x96\xe8\x72\x41\x63\xd9\xab\x72\x98\x24\x8d\x59\xc6\x88\x76\xb4\x7c\x15\x2b\x19\xeb\xec\x6e\x27\x25\xed\x87\xb9\x85\xd6\xf1\xed\x9c\xb0\x23\x8d\xe8\x1b\x8e\x2a\xd0\xad\xb2\xa3\x12\x8e\x3d\x68\x6c\x23\xb7\xed\x07\xd6\x41\xc7\x23\x0e\x81\x63\x51\x83\x3a\xad\x84\xdc\x20\xd4\xeb\xb4\x74\x6a\x2b\xdc\x3e\x89\x7a\xaa\xcf\x6a\xf4\x14\x07\x64\x63\x78\x15\xba\xc2\x10\x92\x5b\xaa\x70\x7d\x65\xbc\x5f\x5b\xb7\x15\x1c\x1f\x29\xb1\xd8\x96\x6d\x21\x9e\x94\x89\xb6\x33\xd2\xbe\x10\x27\x76\x97\xe3\x48\x99\x1c\xbb\x92\xaf\x5f\x7e\x08\x20\x36\xf6\x8a\xee\x41\x7f\x34\x5f\x7c\x34\x83\xd2\xff\x5b\xa2\x23\x9a\x3c\x64\xbc\x36\xdd\x7f\x6b\xb5\x3f\xcb\x06\x4f\x6e\x6c\x0a\x41\x6e\x75\x4c\x37\x4f\xbe\x8c\xf0\x7e\xb6\x40\x87\xb1\xe6\x70\xa3\x88\xe3\x07\x72\x4b\xfe\x1e\xdf\x2c\xed\xcc\xd2\x2c\xad\xbf\xa5\x91\x8f\xae\xf6\xae\xba\x33\xb9\xf3\x19\x77\xab\xff\x21\x9c\x51\x66\x93\x2c\xc6\x58\x66\xa9\x54\xef\xfd\xff\xb1\x31\xb5\xdb\xbf\x01\x00\x00\xff\xff\xe3\x27\xd2\xdd\x7e\x08\x00\x00") +var _pkgUiTemplatesStoresHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x56\x51\x6b\xdb\x30\x10\x7e\xcf\xaf\x38\x4c\x5f\x13\x43\x5f\x06\xc3\xc9\x18\xa3\xb0\x87\xb6\x0c\xd2\xf5\x75\x28\xd6\x25\x16\x55\x24\xa3\x3b\xb7\x09\x42\xff\x7d\xc8\xb1\x13\xbb\x76\xb2\x64\x2f\x06\x9d\xee\xf4\x7d\xd6\xf7\xdd\xd9\xde\x4b\x5c\x2b\x83\x90\x14\x28\x64\x12\xc2\x24\xd3\xca\xbc\x01\xef\x4b\x9c\x27\x8c\x3b\x4e\x73\xa2\x04\x1c\xea\x79\x42\xbc\xd7\x48\x05\x22\x27\x50\x38\x5c\xcf\x13\xef\xa1\x14\x5c\xfc\x72\xb8\x56\x3b\x08\x21\x25\x16\xac\xf2\x58\x93\xba\x4a\x23\xcd\x72\xa2\x6f\xef\x73\xef\x61\x55\x29\x2d\x5f\xd1\x91\xb2\x06\x42\x48\x16\x13\xef\xd1\xc8\x10\x26\x93\x13\x89\xdc\x1a\x46\xc3\x35\x0f\xa9\xde\x21\xd7\x82\x68\x5e\x87\x85\x32\xe8\xa6\x6b\x5d\x29\x99\x2c\x26\x00\x00\xde\x3b\x61\x36\x08\x77\xc4\xd6\xe1\xcb\xbe\x44\xf8\x3a\x87\xd9\xd2\x56\x2e\x47\x0a\xa1\x49\x52\xeb\x4e\x46\x13\xcd\x8a\xfb\x85\xf7\xac\x58\x77\xcb\x67\x4b\x76\xca\x6c\x42\xc8\xd2\xe2\xbe\xc5\x40\x4d\xdd\xaa\xdf\xe6\xcd\xd8\x0f\x03\x31\xbf\x97\x56\xbf\x4a\x9d\xc5\x62\xa5\xb1\xa5\x7e\x58\xd4\xcf\xe9\xca\x3a\x89\x0e\x5b\xfe\x87\xe4\x78\xef\xdd\xb5\x3b\x2d\x9a\x84\xc5\x83\x91\xa5\x55\x86\xb3\x94\x8b\xe1\xee\x92\x05\x57\x34\xbe\xf7\xdd\x18\x5b\x99\x1c\x25\x3c\x8a\x15\xea\x25\xf2\x99\xc4\x27\x65\xe0\x45\x6d\xf1\xcc\xae\xd8\x5d\xd8\x7d\x14\xc4\xf0\x13\x85\xe6\x02\x7e\x14\x98\xbf\x5d\x48\x7b\x42\x22\xb1\xf9\x74\x50\x96\x76\xdf\x3a\xee\x7d\xba\x93\x95\x95\xfb\xd3\xba\xaf\x7b\xd4\x5c\x19\x89\x3b\xb8\x9b\x2d\x63\x80\x86\x72\x9f\xb9\x59\xb9\xf0\xfe\x90\x3b\x7b\x16\x5b\x8c\xba\xb3\x1c\x24\xb5\x4a\x46\x6b\x63\xd2\xdf\x86\xd6\x61\xc6\x72\x03\x3b\x8b\xef\xf9\xe0\x9c\x75\x1d\xf0\xe3\x71\x54\x0a\xd3\x1e\x28\x34\x3a\x86\xfa\x39\xa5\x2a\xcf\x91\x08\x6a\x90\x3f\xca\x48\x95\x0b\xb6\x0e\x62\x07\x4e\xab\xb2\x44\x97\x0b\x1a\x43\xaf\xca\x21\x48\x1a\x51\xc6\x88\x76\xbc\x7c\x15\x2b\x19\xef\xd9\xdd\x4e\x4a\xda\x0f\x73\x0b\xad\x63\xef\x9c\x72\x47\x84\x18\x96\xde\xd4\x6a\xa7\xa2\xbe\x9d\x4e\x3c\x1a\x5b\xe9\xb6\x55\xa2\xb5\x8e\xa2\x36\xb1\xb1\xeb\xfb\xec\xac\x8b\x9c\x87\x50\x35\xcc\x11\xf4\x00\x35\x86\x73\x3c\xb6\x2b\xd7\x4a\xc8\x0d\x42\xfd\x9c\x96\x4e\x6d\x85\xdb\x27\xd1\xd6\xf5\x79\x8d\xad\xe3\x9c\x6e\x02\xaf\x42\x57\x18\x42\x72\x4e\x0c\x38\x23\xc8\x11\x7b\x20\x0c\x0c\x1a\xf8\x5f\xe7\x64\xe9\x19\x05\xb2\xb4\x16\x6f\x71\x85\x15\xbc\x5f\x5b\xb7\x15\x1c\xa7\x12\xb1\xd8\x96\xad\x50\x4f\xca\xc4\xd8\x99\x5e\xbe\x50\x27\x76\x97\xeb\x48\x99\x1c\xbb\x3d\x5e\x8f\xba\x10\x40\x6c\xec\x95\x76\xed\x7c\x8b\x2e\x4e\x09\xb8\xbd\x27\x47\x7c\x7e\x40\xbc\x16\xee\xbf\x9b\xb3\xaf\xfd\x60\xc6\x8c\x8d\x5d\xc8\xad\x8e\x70\xf3\xe4\xcb\x08\xef\x67\x0b\x74\x98\xe3\x0e\x37\x8a\x38\xb6\xf1\x2d\xf8\x3d\xbe\x3d\xaf\xf5\xfc\x35\x64\xba\xea\x7e\x84\x3a\x7f\x1f\xdd\xdb\xff\x10\xce\x28\xb3\x49\x16\x63\x2c\xb3\x54\xaa\xf7\xfe\x0f\x41\x13\x6a\x97\x7f\x03\x00\x00\xff\xff\xb1\x86\x1f\x43\x6f\x09\x00\x00") func pkgUiTemplatesStoresHtmlBytes() ([]byte, error) { return bindataRead( @@ -340,7 +340,7 @@ func pkgUiTemplatesStoresHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/stores.html", size: 2174, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/stores.html", size: 2415, mode: os.FileMode(436), modTime: time.Unix(1573923761, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -360,7 +360,7 @@ func pkgUiStaticCssAlertsCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/alerts.css", size: 401, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/alerts.css", size: 401, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -380,7 +380,7 @@ func pkgUiStaticCssGraphCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/graph.css", size: 3844, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/graph.css", size: 3844, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -400,7 +400,7 @@ func pkgUiStaticCssPrometheusCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/prometheus.css", size: 470, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/prometheus.css", size: 470, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -420,7 +420,7 @@ func pkgUiStaticCssRulesCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/rules.css", size: 195, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/rules.css", size: 195, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -440,7 +440,7 @@ func pkgUiStaticImgAjaxLoaderGif() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -460,7 +460,7 @@ func pkgUiStaticImgFaviconIco() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/img/favicon.ico", size: 15886, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/img/favicon.ico", size: 15886, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -480,7 +480,7 @@ func pkgUiStaticJsAlertsJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 1152, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 1152, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -500,7 +500,7 @@ func pkgUiStaticJsBucketJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/bucket.js", size: 2834, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/bucket.js", size: 2834, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -520,7 +520,7 @@ func pkgUiStaticJsGraphJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 37987, mode: os.FileMode(420), modTime: time.Unix(1571324761, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 37987, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -540,7 +540,7 @@ func pkgUiStaticJsGraph_templateHandlebar() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/graph_template.handlebar", size: 8984, mode: os.FileMode(420), modTime: time.Unix(1570545416, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/graph_template.handlebar", size: 8984, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -560,7 +560,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapGridCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.css", size: 37644, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.css", size: 37644, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -580,7 +580,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapGridMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.min.css", size: 28977, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.min.css", size: 28977, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -600,7 +600,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapRebootCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.css", size: 4896, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.css", size: 4896, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -620,7 +620,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapRebootMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.min.css", size: 4019, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.min.css", size: 4019, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -640,7 +640,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap.min.css", size: 140936, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap.min.css", size: 140936, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -660,7 +660,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapBundleJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.js", size: 212345, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.js", size: 212345, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -680,7 +680,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapBundleMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.min.js", size: 70966, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.min.js", size: 70966, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -700,7 +700,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.min.js", size: 51039, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.min.js", size: 51039, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -720,7 +720,7 @@ func pkgUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs() (*asset, err return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 11273, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 11273, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -740,7 +740,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsCssBootstrapGlyphiconsCss() (*asset, e return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.css", size: 14523, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.css", size: 14523, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -760,7 +760,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsCssBootstrapGlyphiconsMinCss() (*asset return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css", size: 11830, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css", size: 11830, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -780,7 +780,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Eot() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.eot", size: 98620, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.eot", size: 98620, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -800,7 +800,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Svg() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.svg", size: 507478, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.svg", size: 507478, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -820,7 +820,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Ttf() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.ttf", size: 98384, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.ttf", size: 98384, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -840,7 +840,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Woff() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff", size: 63712, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff", size: 63712, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -860,7 +860,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Woff2() (*a return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff2", size: 54420, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff2", size: 54420, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -880,7 +880,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Eot() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.eot", size: 31156, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.eot", size: 31156, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -900,7 +900,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Svg() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.svg", size: 107199, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.svg", size: 107199, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -920,7 +920,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Ttf() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.ttf", size: 30928, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.ttf", size: 30928, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -940,7 +940,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Woff() (*a return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff", size: 14712, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff", size: 14712, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -960,7 +960,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Woff2() (* return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff2", size: 12220, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff2", size: 12220, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -980,7 +980,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Eot() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.eot", size: 102152, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.eot", size: 102152, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1000,7 +1000,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Svg() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.svg", size: 378215, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.svg", size: 378215, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1020,7 +1020,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Ttf() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.ttf", size: 101932, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.ttf", size: 101932, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1040,7 +1040,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Woff() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff", size: 48704, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff", size: 48704, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1060,7 +1060,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Woff2() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff2", size: 38784, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff2", size: 38784, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1080,7 +1080,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.eot", size: 20127, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.eot", size: 20127, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1100,7 +1100,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.svg", size: 108738, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.svg", size: 108738, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1120,7 +1120,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.ttf", size: 45404, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.ttf", size: 45404, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1140,7 +1140,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff", size: 23424, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff", size: 23424, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1160,7 +1160,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff2", size: 18028, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff2", size: 18028, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1180,7 +1180,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeCss() (*asset return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.css", size: 51062, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.css", size: 51062, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1200,7 +1200,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeLess() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.less", size: 53867, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.less", size: 53867, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1220,7 +1220,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeMinCss() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.min.css", size: 42307, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.min.css", size: 42307, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1240,7 +1240,7 @@ func pkgUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinC return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1260,7 +1260,7 @@ func pkgUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinJ return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1280,7 +1280,7 @@ func pkgUiStaticVendorFuzzyFuzzyJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/fuzzy/fuzzy.js", size: 5669, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/fuzzy/fuzzy.js", size: 5669, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1300,7 +1300,7 @@ func pkgUiStaticVendorJsJquery331MinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery-3.3.1.min.js", size: 86927, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery-3.3.1.min.js", size: 86927, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1320,7 +1320,7 @@ func pkgUiStaticVendorJsJqueryHotkeysJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.hotkeys.js", size: 4490, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.hotkeys.js", size: 4490, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1340,7 +1340,7 @@ func pkgUiStaticVendorJsJqueryMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.min.js", size: 86671, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.min.js", size: 86671, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1360,7 +1360,7 @@ func pkgUiStaticVendorJsJquerySelectionJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.selection.js", size: 12881, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.selection.js", size: 12881, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1380,7 +1380,7 @@ func pkgUiStaticVendorJsPopperMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/popper.min.js", size: 19236, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/popper.min.js", size: 19236, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1400,7 +1400,7 @@ func pkgUiStaticVendorMomentMomentTimezoneWithDataMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment-timezone-with-data.min.js", size: 184495, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment-timezone-with-data.min.js", size: 184495, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1420,7 +1420,7 @@ func pkgUiStaticVendorMomentMomentMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment.min.js", size: 51825, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment.min.js", size: 51825, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1440,7 +1440,7 @@ func pkgUiStaticVendorMustacheMustacheMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1460,7 +1460,7 @@ func pkgUiStaticVendorRickshawRickshawMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1480,7 +1480,7 @@ func pkgUiStaticVendorRickshawRickshawMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1500,7 +1500,7 @@ func pkgUiStaticVendorRickshawVendorD3LayoutMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1520,7 +1520,7 @@ func pkgUiStaticVendorRickshawVendorD3V3Js() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(420), modTime: time.Unix(1568887383, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1810,7 +1810,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ }}, }} -// RestoreAsset restores an asset under the given directory. +// RestoreAsset restores an asset under the given directory func RestoreAsset(dir, name string) error { data, err := Asset(name) if err != nil { @@ -1835,14 +1835,14 @@ func RestoreAsset(dir, name string) error { return nil } -// RestoreAssets restores an asset under the given directory recursively. +// RestoreAssets restores an asset under the given directory recursively func RestoreAssets(dir, name string) error { children, err := AssetDir(name) - // File. + // File if err != nil { return RestoreAsset(dir, name) } - // Dir. + // Dir for _, child := range children { err = RestoreAssets(dir, filepath.Join(name, child)) if err != nil { diff --git a/pkg/ui/templates/stores.html b/pkg/ui/templates/stores.html index 316553648e..c4cb297493 100644 --- a/pkg/ui/templates/stores.html +++ b/pkg/ui/templates/stores.html @@ -28,21 +28,25 @@

Unknown Type

{{$store.Name}} {{if not $store.LastError}} - - up - + up {{else}} - - down - + down {{end}} - {{range $labelSets := $store.LabelSets}} - {{range $label := $labelSets.Labels}} - {{$label.Name}}="{{$label.Value}}" + + + {{range $labelSets := $store.LabelSets}} + + + {{end}} - {{end}} + +
+ {{range $label := $labelSets.Labels}} + {{$label.Name}}="{{$label.Value}}" + {{end}} +
{{formatTimestamp $store.MinTime}} {{formatTimestamp $store.MaxTime}} From f840e1855c9ecb5c63383d4f0a85180c6131edd7 Mon Sep 17 00:00:00 2001 From: Antonio Santos Date: Tue, 19 Nov 2019 10:07:33 +0100 Subject: [PATCH 052/257] .*: Add support for tracing with Lightstep (#1678) * Add support for tracing with Lightstep Signed-off-by: Antonio Santos * Remove type cast Signed-off-by: Antonio Santos * Add documentation for the exported methods and types Signed-off-by: Antonio Santos * Apply suggestions from @bwplotka Co-Authored-By: Bartlomiej Plotka Signed-off-by: Antonio Santos * Inline variable Co-Authored-By: Povilas Versockas Signed-off-by: Antonio Santos * Update lightstep library and add proper error handling https://github.com/lightstep/lightstep-tracer-go/pull/228 introduced a new method to create a Tracer that returns an error when it does not succeed, instead of nil Signed-off-by: Antonio Santos * Remove unused import Signed-off-by: Antonio Santos --- CHANGELOG.md | 1 + docs/tracing.md | 19 +++ go.mod | 1 + go.sum | 189 ++++++----------------------- pkg/tracing/client/factory.go | 4 + pkg/tracing/lightstep/lightstep.go | 57 +++++++++ scripts/cfggen/main.go | 2 + 7 files changed, 119 insertions(+), 154 deletions(-) create mode 100644 pkg/tracing/lightstep/lightstep.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7301a46339..b3f0ede657 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Added +- [#1678](https://github.com/thanos-io/thanos/pull/1678) Add Lightstep as a tracing provider. - [#1687](https://github.com/thanos-io/thanos/pull/1687) Add a new `--grpc-grace-period` CLI option to components which serve gRPC to set how long to wait until gRPC Server shuts down. - [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. - [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss) for further information. diff --git a/docs/tracing.md b/docs/tracing.md index cfc0542974..1191bec088 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -101,3 +101,22 @@ config: service_version: "" service_environment: "" ``` + +### Lightstep + +Client for [Ligthstep](https://lightstep.com). + +In order to configure Thanos to interact with Lightstep you need to provide at least an [access token](https://docs.lightstep.com/docs/create-and-use-access-tokens) in the configuration file. The `collector` key is optional and used when you have on-premise satellites. + +[embedmd]:# (flags/config_tracing_lightstep.txt yaml) +```yaml +type: LIGHTSTEP +config: + access_token: "" + collector: + scheme: "" + host: "" + port: 0 + plaintext: false + custom_ca_cert_file: "" +``` diff --git a/go.mod b/go.mod index de1d4ed9b9..17c51658cf 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/leanovate/gopter v0.2.4 + github.com/lightstep/lightstep-tracer-go v0.18.0 github.com/lovoo/gcloud-opentracing v0.3.0 github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe // indirect github.com/mattn/go-runewidth v0.0.6 // indirect diff --git a/go.sum b/go.sum index f049dfe801..598db171d8 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,24 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1 h1:7gXaI3V/b4DRaK++rTqhRajcT7z8gtP0qKMZTXqlySM= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.48.0 h1:6ZHYIRlohUdU4LrLHbTsReY1eYy/MoZW1FsEyBuMXsk= cloud.google.com/go v0.48.0/go.mod h1:gGOnoa/XMQYHAscREBlbdHduGchEaP9N0//OXdrPI/M= -cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.3.0 h1:2Ze/3nQD5F+HfL0xOPM2EeawDWs+NPRtzgcre+17iZU= cloud.google.com/go/storage v1.3.0/go.mod h1:9IAwXhoyBJ7z9LcAwkj0/7NnPzYaPeZxxVp3zm+5IqA= -contrib.go.opencensus.io/exporter/ocagent v0.6.0 h1:Z1n6UAyr0QwM284yUuh5Zd8JlvxUGAhFZcgMJkMPrGM= contrib.go.opencensus.io/exporter/ocagent v0.6.0/go.mod h1:zmKjrJcdo0aYcVS7bmEeSEBLPA9YJp5bjrofdU3pIXs= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= -github.com/Azure/azure-sdk-for-go v23.2.0+incompatible h1:bch1RS060vGpHpY3zvQDV4rOiRw25J1zmR/B9a76aSA= github.com/Azure/azure-sdk-for-go v23.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v36.1.0+incompatible h1:smHlbChr/JDmsyUqELZXLs0YIgpXecIGdUibuc2983s= github.com/Azure/azure-sdk-for-go v36.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -48,7 +37,6 @@ github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSW github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/to v0.3.1-0.20191028180845-3492b2aff503 h1:2McfZNaDqGPjv2pddK547PENIk4HV+NT7gvqRq4L0us= github.com/Azure/go-autorest/autorest/to v0.3.1-0.20191028180845-3492b2aff503/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= @@ -58,33 +46,29 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.6 h1:U68crOE3y3MPttCMQGywZOLrTeF5HHJ3/vDBCJn9/bA= github.com/OneOfOne/xxhash v1.2.6/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible h1:EaK5256H3ELiyaq5O/Zwd6fnghD6DqmZDQmmzzJklUU= github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.0 h1:B7AQgHi8QSEi4uHu7Sbsga+IJDU+CENgjxoo81vDUqU= github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs= @@ -93,38 +77,30 @@ github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.23.12 h1:2UnxgNO6Y5J1OrkXS8XNp0UatDxD1bWHiDT62RDPggI= github.com/aws/aws-sdk-go v1.23.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.35 h1:fe2tJnqty/v/50pyngKdNk/NP8PFphYDA1Z7N3EiiiE= github.com/aws/aws-sdk-go v1.25.35/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v0.0.0-20181017004759-096ff4a8a059/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda h1:NyywMz59neOoVRFDz+ccfKWxn784fiHMDnZSy6T+JXY= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -133,27 +109,23 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/go-sysinfo v1.0.1 h1:lzGPX2sIXaETeMXitXL2XZU8K4B7k7JBhIKWxdOdUt8= github.com/elastic/go-sysinfo v1.0.1/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= github.com/elastic/go-sysinfo v1.1.1 h1:ZVlaLDyhVkDfjwPGU55CQRCRolNpc7P0BbyhhQZQmMI= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= -github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= -github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550 h1:mV9jbLoSW/8m4VK16ZkHTozJa8sesK5u5kTMFysTYac= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/structtag v1.1.0 h1:6j4mUV/ES2duvnAzKMFkN6/A5mCaNYPD3xfbAkLLOF8= github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -161,7 +133,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -190,21 +161,16 @@ github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 h1:X+zN6RZXsvnrSJaAIQhZezPfAfvsqihKKR8oiLHid34= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -212,30 +178,24 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -249,81 +209,62 @@ github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= -github.com/gophercloud/gophercloud v0.3.0 h1:6sjpKIpVwRIIwmcEGp+WwNovNsem+c+2vm6oxshRpL8= github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gophercloud/gophercloud v0.6.0 h1:Xb2lcqZtml1XjgYZxbeayEemq7ASbeTp09m36gQFpEU= github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de h1:F7WD09S8QB4LrkEpka0dFPLSotH11HRpCsLIbIcJ7sU= github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1 h1:zCy2xE9ablevUOrUZc3Dl72Dt+ya2FNAvC2yLYMHzi4= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= -github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0 h1:HXNYlRkkM/t+Y/Yhxtwcy02dlYwIaoxzvxPnS+cqy78= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0 h1:UOxjlb4xVNF93jak1mzzoBatyFju9nrkxpVwIp/QqxQ= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.1 h1:DMo4fmknnz0E0evoNYnV48RjWndOsmd6OW+09R3cEP8= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM= github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc= github.com/hashicorp/serf v0.8.5/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= @@ -336,19 +277,13 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -356,34 +291,33 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= -github.com/leanovate/gopter v0.2.4 h1:U4YLBggDFhJdqQsG4Na2zX7joVTky9vHaj/AGEwSuXU= github.com/leanovate/gopter v0.2.4/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743 h1:143Bb8f8DuGWck/xpNUOckBVYfFbBTnLevfRZ1aVVqo= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.17.1 h1:PgitbgUDool2AcHopDNTlvwq7BQeZssTGs4EVwcGhr8= +github.com/lightstep/lightstep-tracer-go v0.17.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lightstep/lightstep-tracer-go v0.18.0 h1:fAazJekOWnfBeQYwk9jEgIWWKmBxq4ev3WfsAnezgc4= +github.com/lightstep/lightstep-tracer-go v0.18.0/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lovoo/gcloud-opentracing v0.3.0 h1:nAeKG70rIsog0TelcEtt6KU0Y1s5qXtsDLnHp0urPLU= github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe h1:YioO2TiJyAHWHyCRQCP8jk5IzTqmsbGc5qQPIhHo6xs= github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.6 h1:V2iyH+aX9C5fsYCpK60U8BYIvmhqxuOL3JZcqc1NB7k= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc= github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= @@ -392,11 +326,9 @@ github.com/minio/minio-go/v6 v6.0.41/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tB github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -414,7 +346,6 @@ github.com/mozillazg/go-cos v0.13.0 h1:RylOpEESdWMLb13bl0ADhko12uMN3JmHqqwFu4OYG github.com/mozillazg/go-cos v0.13.0/go.mod h1:Zp6DvvXn0RUOXGJ2chmWt2bLEqRAnJnS3DnAZsJsoaE= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -424,86 +355,75 @@ github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQ github.com/oklog/ulid v0.0.0-20170117200651-66bb6560562f/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2 h1:sq53g+DWf0J6/ceFUHpQ0nAEb6WgM++fq16MZ91cS6o= github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= -github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3 h1:EooPXg51Tn+xmWPXJUGCnJhJSpeuMlBmfJVcqIRmmv8= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 h1:QsgXACQhd9QJhEmRumbsMQQvBtmdS0mafoVEBplWXEg= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/alertmanager v0.18.0/go.mod h1:WcxHBl40VSPuOaqWae6l6HpnEOVRIycEJ7i9iYkadEE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.2.0 h1:g4yo/h/me4ZL9o0SVHNRdS2jn5SY8GDmMgkhQ8Mz70s= github.com/prometheus/client_golang v1.2.0/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.6 h1:0qbH+Yqu/cj1ViVLvEWCP6qMQ4efWUj6bQqOEA0V0U4= github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/prometheus v1.8.2-0.20191114185310-85cae37107c2 h1:33wX5JpFUQiXXSWO/g8a9MwYj8VzEno7s5bpJpVEtN8= github.com/prometheus/prometheus v1.8.2-0.20191114185310-85cae37107c2/go.mod h1:PVTKYlgELGIDbIKIyWRzD4WKjnavPynGOFLSuDpvOwU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190810000440-0ceca61e4d75 h1:cA+Ubq9qEVIQhIWvP2kNuSZ2CmnfBJFSRq+kO1pu2cc= github.com/samuel/go-zookeeper v0.0.0-20190810000440-0ceca61e4d75/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= -github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef h1:RoeI7K0oZIcUirMHsFpQjTVDrl1ouNh8T7v3eNsUxL0= github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= @@ -511,31 +431,21 @@ github.com/shurcooL/vfsgen v0.0.0-20180825020608-02ddb050ef6b/go.mod h1:TrYk7fJV github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= @@ -553,12 +463,12 @@ go.elastic.co/apm/module/apmot v1.5.0/go.mod h1:d2KYwhJParTpyw2WnTNy8geNlHKKFX+4 go.elastic.co/fastjson v1.0.0 h1:ooXV/ABvf+tBul26jcVViPT3sBir0PvXgibYB1IQQzg= go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -570,13 +480,10 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -584,7 +491,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -592,10 +498,8 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -610,27 +514,24 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -638,9 +539,7 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -648,18 +547,17 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180805044716-cb6730876b98/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -676,56 +574,48 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18 h1:xFbv3LvlvQAmbNJFCBKRv1Ccvnh9FVsW0FX2kTWWowE= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190918214516-5a1a30219888/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191111182352-50fa39b762bc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191114222411-4191b8cbba09 h1:f3ZhcxGnJxVhcsQgKPvfAg2pjdeWBBgDuO5XfmGnoU0= golang.org/x/tools v0.0.0-20191114222411-4191b8cbba09/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0 h1:VGGbLNyPF7dvYHhcUGYBBGCRDDK0RRJAI6KCvo0CL+E= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 h1:iKtrH9Y8mcbADOP0YFaEMth7OfuHY9xHOwNj4znpM1A= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de h1:dFEMUWudT9iV1JMk6i6NwbfIw2V/2VDFyDYCZFypRxE= google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.22.1 h1:/7cs52RnTJmD43s3uxzlq2U7nqVTd/37viQwMrMNlOM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= @@ -734,40 +624,33 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/fsnotify/fsnotify.v1 v1.4.7 h1:XNNYLJHt73EyYiCZi6+xjupS9CpvmiDgjPTAjrBlQbo= gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= -gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= @@ -779,9 +662,7 @@ k8s.io/client-go v0.0.0-20190620085101-78d2af792bab h1:E8Fecph0qbNsAbijJJQryKu4O k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k= k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= -k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6 h1:p0Ai3qVtkbCG/Af26dBmU0E1W58NID3hSSh7cMyylpM= k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= diff --git a/pkg/tracing/client/factory.go b/pkg/tracing/client/factory.go index 6f346a70b6..54f40a7292 100644 --- a/pkg/tracing/client/factory.go +++ b/pkg/tracing/client/factory.go @@ -12,6 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/thanos-io/thanos/pkg/tracing/elasticapm" "github.com/thanos-io/thanos/pkg/tracing/jaeger" + "github.com/thanos-io/thanos/pkg/tracing/lightstep" "github.com/thanos-io/thanos/pkg/tracing/stackdriver" "gopkg.in/yaml.v2" ) @@ -22,6 +23,7 @@ const ( STACKDRIVER TracingProvider = "STACKDRIVER" JAEGER TracingProvider = "JAEGER" ELASTIC_APM TracingProvider = "ELASTIC_APM" + LIGHTSTEP TracingProvider = "LIGHTSTEP" ) type TracingConfig struct { @@ -53,6 +55,8 @@ func NewTracer(ctx context.Context, logger log.Logger, metrics *prometheus.Regis return jaeger.NewTracer(ctx, logger, metrics, config) case string(ELASTIC_APM): return elasticapm.NewTracer(config) + case string(LIGHTSTEP): + return lightstep.NewTracer(ctx, config) default: return nil, nil, errors.Errorf("tracing with type %s is not supported", tracingConf.Type) } diff --git a/pkg/tracing/lightstep/lightstep.go b/pkg/tracing/lightstep/lightstep.go new file mode 100644 index 0000000000..c6c3584905 --- /dev/null +++ b/pkg/tracing/lightstep/lightstep.go @@ -0,0 +1,57 @@ +package lightstep + +import ( + "context" + "io" + + "github.com/lightstep/lightstep-tracer-go" + "github.com/opentracing/opentracing-go" + "gopkg.in/yaml.v2" +) + +// Config - YAML configuration. +type Config struct { + // AccessToken is the unique API key for your LightStep project. It is + // available on your account page at https://app.lightstep.com/account. + AccessToken string `yaml:"access_token"` + + // Collector is the host, port, and plaintext option to use + // for the collector. + Collector lightstep.Endpoint `yaml:"collector"` +} + +// Tracer wraps the Lightstep tracer and the context. +type Tracer struct { + lightstep.Tracer + ctx context.Context +} + +// Close synchronously flushes the Lightstep tracer, then terminates it. +func (t *Tracer) Close() error { + t.Tracer.Close(t.ctx) + + return nil +} + +// NewTracer creates a Tracer with the options present in the YAML config. +func NewTracer(ctx context.Context, yamlConfig []byte) (opentracing.Tracer, io.Closer, error) { + config := Config{} + if err := yaml.Unmarshal(yamlConfig, &config); err != nil { + return nil, nil, err + } + + options := lightstep.Options{ + AccessToken: config.AccessToken, + Collector: config.Collector, + } + lighstepTracer, err := lightstep.CreateTracer(options) + if err != nil { + return nil, nil, err + } + + t := &Tracer{ + lighstepTracer, + ctx, + } + return t, t, nil +} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index ca9e1933f9..9051848a38 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -23,6 +23,7 @@ import ( trclient "github.com/thanos-io/thanos/pkg/tracing/client" "github.com/thanos-io/thanos/pkg/tracing/elasticapm" "github.com/thanos-io/thanos/pkg/tracing/jaeger" + "github.com/thanos-io/thanos/pkg/tracing/lightstep" "github.com/thanos-io/thanos/pkg/tracing/stackdriver" kingpin "gopkg.in/alecthomas/kingpin.v2" yaml "gopkg.in/yaml.v2" @@ -42,6 +43,7 @@ var ( trclient.JAEGER: jaeger.Config{}, trclient.STACKDRIVER: stackdriver.Config{}, trclient.ELASTIC_APM: elasticapm.Config{}, + trclient.LIGHTSTEP: lightstep.Config{}, } ) From c7fc04e3dde9da775c29ebf7cd9e87aa0f4309ee Mon Sep 17 00:00:00 2001 From: Wade Rossmann Date: Tue, 19 Nov 2019 16:33:33 -0800 Subject: [PATCH 053/257] bucket: Add `--web.external-prefix` for proxying on a subpath (#1758) * bucket: Add `--web.external-prefix` for proxying on a subpath Signed-off-by: Wade Rossmann * Update PR number from previous broken branch. Signed-off-by: Wade Rossmann * Linter fixes Signed-off-by: Wade Rossmann * Doc fixes Signed-off-by: Wade Rossmann --- CHANGELOG.md | 1 + cmd/thanos/bucket.go | 7 ++++- docs/components/bucket.md | 59 ++++++++++++++++++++++----------------- pkg/ui/bucket.go | 13 +++++---- 4 files changed, 48 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3f0ede657..b5acb9bffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1680](https://github.com/thanos-io/thanos/pull/1680) Add a new `--http-grace-period` CLI option to components which serve HTTP to set how long to wait until HTTP Server shuts down. - [#1712](https://github.com/thanos-io/thanos/pull/1712) Rename flag on bucket web component from `--listen` to `--http-address` to match other components. - [#1733](https://github.com/thanos-io/thanos/pull/1733) New metric `thanos_compactor_iterations_total` on Thanos Compactor which shows the number of successful iterations. +- [#1758](https://github.com/thanos-io/thanos/pull/1758) `thanos bucket web` now supports `--web.external-prefix` for proxying on a subpath. ### Fixed diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 3a7646501c..6581e154d5 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -314,6 +314,7 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str interval := cmd.Flag("refresh", "Refresh interval to download metadata from remote storage").Default("30m").Duration() timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() label := cmd.Flag("label", "Prometheus label to use as timeline title").String() + webExternalPrefix := cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the UI query web interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos UI to be served behind a reverse proxy that strips a URL sub-path.").Default("").String() m[name+" web"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { ctx, cancel := context.WithCancel(context.Background()) @@ -325,8 +326,12 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str httpserver.WithGracePeriod(time.Duration(*httpGracePeriod)), ) + flagsMap := map[string]string{ + "web.external-prefix": *webExternalPrefix, + } + router := route.New() - bucketUI := ui.NewBucketUI(logger, *label) + bucketUI := ui.NewBucketUI(logger, *label, flagsMap) bucketUI.Register(router, extpromhttp.NewInstrumentationMiddleware(reg)) srv.Handle("/", router) diff --git a/docs/components/bucket.md b/docs/components/bucket.md index edcff62cfa..fb977744b0 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -98,37 +98,44 @@ usage: thanos bucket web [] Web interface for remote storage bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + -h, --help Show context-sensitive help (also try + --help-long and --help-man). + --version Show application version. + --log.level=info Log filtering level. + --log.format=logfmt Log format to use. --tracing.config-file= - Path to YAML file with tracing configuration. See - format details: - https://thanos.io/tracing.md/#configuration + Path to YAML file with tracing configuration. + See format details: + https://thanos.io/tracing.md/#configuration --tracing.config= - Alternative to 'tracing.config-file' flag (lower - priority). Content of YAML file with tracing - configuration. See format details: - https://thanos.io/tracing.md/#configuration + Alternative to 'tracing.config-file' flag (lower + priority). Content of YAML file with tracing + configuration. See format details: + https://thanos.io/tracing.md/#configuration --objstore.config-file= - Path to YAML file that contains object store - configuration. See format details: - https://thanos.io/storage.md/#configuration + Path to YAML file that contains object store + configuration. See format details: + https://thanos.io/storage.md/#configuration --objstore.config= - Alternative to 'objstore.config-file' flag (lower - priority). Content of YAML file that contains - object store configuration. See format details: - https://thanos.io/storage.md/#configuration + Alternative to 'objstore.config-file' flag + (lower priority). Content of YAML file that + contains object store configuration. See format + details: + https://thanos.io/storage.md/#configuration --http-address="0.0.0.0:10902" - Listen host:port for HTTP endpoints. - --http-grace-period=2m Time to wait after an interrupt received for HTTP - Server. - --refresh=30m Refresh interval to download metadata from remote - storage - --timeout=5m Timeout to download metadata from remote storage - --label=LABEL Prometheus label to use as timeline title + Listen host:port for HTTP endpoints. + --http-grace-period=2m Time to wait after an interrupt received for + HTTP Server. + --refresh=30m Refresh interval to download metadata from + remote storage + --timeout=5m Timeout to download metadata from remote storage + --label=LABEL Prometheus label to use as timeline title + --web.external-prefix="" Static prefix for all HTML links and redirect + URLs in the UI query web interface. Actual + endpoints are still served on / or the + web.route-prefix. This allows thanos UI to be + served behind a reverse proxy that strips a URL + sub-path. ``` diff --git a/pkg/ui/bucket.go b/pkg/ui/bucket.go index 8e023ba61b..ae9fb2ad41 100644 --- a/pkg/ui/bucket.go +++ b/pkg/ui/bucket.go @@ -19,13 +19,15 @@ type Bucket struct { Blocks template.JS RefreshedAt time.Time Err error + flagsMap map[string]string } -func NewBucketUI(logger log.Logger, label string) *Bucket { +func NewBucketUI(logger log.Logger, label string, flagsMap map[string]string) *Bucket { return &Bucket{ - BaseUI: NewBaseUI(logger, "bucket_menu.html", queryTmplFuncs()), - Blocks: "[]", - Label: label, + BaseUI: NewBaseUI(logger, "bucket_menu.html", queryTmplFuncs()), + Blocks: "[]", + Label: label, + flagsMap: flagsMap, } } @@ -41,7 +43,8 @@ func (b *Bucket) Register(r *route.Router, ins extpromhttp.InstrumentationMiddle // Handle / of bucket UIs. func (b *Bucket) root(w http.ResponseWriter, r *http.Request) { - b.executeTemplate(w, "bucket.html", "", b) + prefix := GetWebPrefix(b.logger, b.flagsMap, r) + b.executeTemplate(w, "bucket.html", prefix, b) } func (b *Bucket) Set(data string, err error) { From c1d7c2fd77e57773be2b3c56c965dbfc81a39d6c Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Wed, 20 Nov 2019 15:25:41 +0100 Subject: [PATCH 054/257] cmd/thanos/rule: simplify if block (#1761) Signed-off-by: Simon Pasquier --- cmd/thanos/rule.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 3d0345b2b0..b34016376f 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -567,13 +567,7 @@ func runRule( return err } - uploads := true - if len(confContentYaml) == 0 { - level.Info(logger).Log("msg", "No supported bucket was configured, uploads will be disabled") - uploads = false - } - - if uploads { + if len(confContentYaml) > 0 { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Rule.String()) @@ -604,6 +598,8 @@ func runRule( }, func(error) { cancel() }) + } else { + level.Info(logger).Log("msg", "no supported bucket was configured, uploads will be disabled") } level.Info(logger).Log("msg", "starting rule node") @@ -727,7 +723,7 @@ func removeDuplicateQueryAddrs(logger log.Logger, duplicatedQueriers prometheus. set := make(map[string]struct{}) for _, addr := range addrs { if _, ok := set[addr]; ok { - level.Warn(logger).Log("msg", "Duplicate query address is provided - %v", addr) + level.Warn(logger).Log("msg", "duplicate query address is provided - %v", addr) duplicatedQueriers.Inc() } set[addr] = struct{}{} From a1c88ba174676b51571a74213a0e7d0ac6c5ff49 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Wed, 20 Nov 2019 14:47:37 +0000 Subject: [PATCH 055/257] Moved release to Thursday; Cleanup tests. (#1763) * Moved release to Thursday. Signed-off-by: Bartek Plotka * Simplified tests. Signed-off-by: Bartek Plotka --- docs/release-process.md | 3 ++- pkg/store/bucket_e2e_test.go | 40 ++++++------------------------------ 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/docs/release-process.md b/docs/release-process.md index 6b25b89abf..dc61ac1e33 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -30,7 +30,8 @@ Release shepherd responsibilities: | Release | Time of first RC | Shepherd (Github handle) | |-----------|--------------------------|--------------------------| -| v0.9.0 | (planned) 20.11.2019 | TBC | +| v0.10.0 | (planned) 8.01.2019 | TBD | +| v0.9.0 | (planned) 21.11.2019 | `@bwplotka` | | v0.8.0 | (planned) 9.10.2019 | `@bwplotka` | | v0.7.0 | (planned) 28.08.2019 | `@domgreen` | | v0.6.0 | (planned) 12.07.2019 | `@GiedriusS` | diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index 43ce3c50b2..f6eecd4cd9 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -5,13 +5,11 @@ import ( "io/ioutil" "os" "path/filepath" - "sync" "testing" "time" "github.com/go-kit/kit/log" "github.com/oklog/ulid" - "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/tsdb/labels" @@ -21,7 +19,6 @@ import ( "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/objstore/objtesting" - "github.com/thanos-io/thanos/pkg/runutil" storecache "github.com/thanos-io/thanos/pkg/store/cache" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" @@ -70,9 +67,6 @@ func (c *swappableCache) Series(b ulid.ULID, id uint64) ([]byte, bool) { } type storeSuite struct { - cancel context.CancelFunc - wg sync.WaitGroup - store *BucketStore minTime, maxTime int64 cache *swappableCache @@ -80,11 +74,6 @@ type storeSuite struct { logger log.Logger } -func (s *storeSuite) Close() { - s.cancel() - s.wg.Wait() -} - func prepareTestBlocks(t testing.TB, now time.Time, count int, dir string, bkt objstore.Bucket, series []labels.Labels, extLset labels.Labels) (blocks int, minTime, maxTime int64) { ctx := context.Background() @@ -142,9 +131,7 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m blocks, minTime, maxTime := prepareTestBlocks(t, time.Now(), 3, dir, bkt, series, extLset) - ctx, cancel := context.WithCancel(context.Background()) s := &storeSuite{ - cancel: cancel, logger: log.NewLogfmtLogger(os.Stderr), cache: &swappableCache{}, minTime: minTime, @@ -159,27 +146,14 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m s.store.partitioner = naivePartitioner{} } - s.wg.Add(1) - go func() { - defer s.wg.Done() - - if err := runutil.Repeat(100*time.Millisecond, ctx.Done(), func() error { - return store.SyncBlocks(ctx) - }); err != nil && ctx.Err() == nil { - t.Fatal(err) - } - }() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() - rctx, rcancel := context.WithTimeout(ctx, 30*time.Second) - defer rcancel() - - testutil.Ok(t, runutil.Retry(100*time.Millisecond, rctx.Done(), func() error { - if store.numBlocks() < blocks { - return errors.New("not all blocks loaded") - } - return nil - })) + testutil.Ok(t, store.SyncBlocks(ctx)) + if store.numBlocks() < blocks { + t.Fatalf("not all blocks loaded got %v, expected %v", store.numBlocks(), blocks) + } return s } @@ -393,7 +367,6 @@ func TestBucketStore_e2e(t *testing.T) { defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() s := prepareStoreWithTestBlocks(t, dir, bkt, false, 0, emptyRelabelConfig) - defer s.Close() t.Log("Test with no index cache") s.cache.SwapWith(noopCache{}) @@ -442,7 +415,6 @@ func TestBucketStore_ManyParts_e2e(t *testing.T) { defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() s := prepareStoreWithTestBlocks(t, dir, bkt, true, 0, emptyRelabelConfig) - defer s.Close() indexCache, err := storecache.NewIndexCache(s.logger, nil, storecache.Opts{ MaxItemSizeBytes: 1e5, From af2a342ba5855ebeca4b5ee179a8608fce874b33 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Wed, 20 Nov 2019 16:00:25 +0100 Subject: [PATCH 056/257] Re-adjust histogram buckets (#1769) Signed-off-by: Kemal Akkoyun --- cmd/thanos/query.go | 2 +- pkg/compact/compact.go | 4 ++-- pkg/extprom/http/instrument_server.go | 2 +- pkg/objstore/objstore.go | 2 +- pkg/server/grpc/grpc.go | 2 +- pkg/store/bucket.go | 4 ++-- pkg/store/gate.go | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 3d612d5fdd..2c836a5b81 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -168,7 +168,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { func storeClientGRPCOpts(logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, secure bool, cert, key, caCert, serverName string) ([]grpc.DialOption, error) { grpcMets := grpc_prometheus.NewClientMetrics() grpcMets.EnableClientHandlingTimeHistogram( - grpc_prometheus.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 15)), + grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), ) dialOpts := []grpc.DialOption{ // We want to make sure that we can receive huge gRPC messages from storeAPI. diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index fbeb9f3b0e..f5d932b386 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -83,7 +83,7 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { m.syncMetaDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "thanos_compact_sync_meta_duration_seconds", Help: "Time it took to sync meta files.", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720}, }) m.garbageCollectedBlocks = prometheus.NewCounter(prometheus.CounterOpts{ @@ -101,7 +101,7 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { m.garbageCollectionDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "thanos_compact_garbage_collection_duration_seconds", Help: "Time it took to perform garbage collection iteration.", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720}, }) m.compactions = prometheus.NewCounterVec(prometheus.CounterOpts{ diff --git a/pkg/extprom/http/instrument_server.go b/pkg/extprom/http/instrument_server.go index 3683592f07..aaef94693b 100644 --- a/pkg/extprom/http/instrument_server.go +++ b/pkg/extprom/http/instrument_server.go @@ -41,7 +41,7 @@ func NewInstrumentationMiddleware(reg prometheus.Registerer) InstrumentationMidd prometheus.HistogramOpts{ Name: "http_request_duration_seconds", Help: "Tracks the latencies for HTTP requests.", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 17), + Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }, []string{"code", "handler", "method"}, ), diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index 4f8ae11f40..f328019b07 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -204,7 +204,7 @@ func BucketWithMetrics(name string, b Bucket, r prometheus.Registerer) Bucket { Name: "thanos_objstore_bucket_operation_duration_seconds", Help: "Duration of operations against the bucket", ConstLabels: prometheus.Labels{"bucket": name}, - Buckets: prometheus.ExponentialBuckets(0.001, 2, 17), + Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }, []string{"operation"}), lastSuccessfullUploadTime: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "thanos_objstore_bucket_last_successful_upload_time", diff --git a/pkg/server/grpc/grpc.go b/pkg/server/grpc/grpc.go index 725479d797..21416b8fda 100644 --- a/pkg/server/grpc/grpc.go +++ b/pkg/server/grpc/grpc.go @@ -43,7 +43,7 @@ func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer met := grpc_prometheus.NewServerMetrics() met.EnableHandlingTimeHistogram( - grpc_prometheus.WithHistogramBuckets(prometheus.ExponentialBuckets(0.001, 2, 15)), + grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), ) panicsTotal := prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_grpc_req_panics_recovered_total", diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index a087ba7bd2..0c924fe376 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -137,12 +137,12 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { m.seriesGetAllDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "thanos_bucket_store_series_get_all_duration_seconds", Help: "Time it takes until all per-block prepares and preloads for a query are finished.", - Buckets: prometheus.ExponentialBuckets(0.01, 2, 15), + Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }) m.seriesMergeDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "thanos_bucket_store_series_merge_duration_seconds", Help: "Time it takes to merge sub-results from all queried blocks into a single result.", - Buckets: prometheus.ExponentialBuckets(0.01, 2, 15), + Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }) m.resultSeriesCount = prometheus.NewSummary(prometheus.SummaryOpts{ Name: "thanos_bucket_store_series_result_series", diff --git a/pkg/store/gate.go b/pkg/store/gate.go index 1a8fe69108..526a3e39a7 100644 --- a/pkg/store/gate.go +++ b/pkg/store/gate.go @@ -26,7 +26,7 @@ func NewGate(maxConcurrent int, reg prometheus.Registerer) *Gate { gateTiming: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "gate_duration_seconds", Help: "How many seconds it took for queries to wait at the gate.", - Buckets: prometheus.ExponentialBuckets(0.1, 2, 15), + Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720}, }), } From 8a8f576a533950e55a0cdb97ab86fd7df3616ea0 Mon Sep 17 00:00:00 2001 From: cmanzi <54186611+cmanzi@users.noreply.github.com> Date: Thu, 21 Nov 2019 22:53:21 -0800 Subject: [PATCH 057/257] Fix web.external-prefix 404s and add web.prefix-header for bucket web UI. (#1770) * Add --web.prefix-header flag. Signed-off-by: Christopher Manzi * Update changelog. Signed-off-by: Christopher Manzi * Resolve conflicts with upstream. Signed-off-by: Christopher Manzi * Lint docs. Signed-off-by: Christopher Manzi * More docs linting. Signed-off-by: Christopher Manzi * Remove extra flags. Signed-off-by: Christopher Manzi * Update changelog. Signed-off-by: Christopher Manzi * Lint docs. Signed-off-by: Christopher Manzi * Final docs lint. Signed-off-by: Christopher Manzi --- CHANGELOG.md | 2 ++ cmd/thanos/bucket.go | 7 +++++-- docs/components/bucket.md | 25 +++++++++++++++++++------ pkg/ui/bucket.go | 2 +- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5acb9bffa..c24b3c09a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1712](https://github.com/thanos-io/thanos/pull/1712) Rename flag on bucket web component from `--listen` to `--http-address` to match other components. - [#1733](https://github.com/thanos-io/thanos/pull/1733) New metric `thanos_compactor_iterations_total` on Thanos Compactor which shows the number of successful iterations. - [#1758](https://github.com/thanos-io/thanos/pull/1758) `thanos bucket web` now supports `--web.external-prefix` for proxying on a subpath. +- [#1770](https://github.com/thanos-io/thanos/pull/1770) Add `--web.prefix-header` flags to allow for bucket UI to be accessible behind a reverse proxy. ### Fixed @@ -28,6 +29,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1669](https://github.com/thanos-io/thanos/pull/1669) Fixed store sharding. Now it does not load excluded meta.jsons and load/fetch index-cache.json files. - [#1670](https://github.com/thanos-io/thanos/pull/1670) Fixed un-ordered blocks upload. Sidecar now uploads the oldest blocks first. - [#1568](https://github.com/thanos-io/thanos/pull/1709) Thanos Store now retains the first raw value of a chunk during downsampling to avoid losing some counter resets that occur on an aggregation boundary. +- [#1770](https://github.com/thanos-io/thanos/pull/1770) Fix `--web.external-prefix` 404s for static resources. ### Changed diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 6581e154d5..0f7c43cc94 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -311,10 +311,11 @@ func registerBucketInspect(m map[string]setupFunc, root *kingpin.CmdClause, name func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) { cmd := root.Command("web", "Web interface for remote storage bucket") httpBindAddr, httpGracePeriod := regHTTPFlags(cmd) + webExternalPrefix := cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the bucket web UI interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos bucket web UI to be served behind a reverse proxy that strips a URL sub-path.").Default("").String() + webPrefixHeaderName := cmd.Flag("web.prefix-header", "Name of HTTP request header used for dynamic prefixing of UI links and redirects. This option is ignored if web.external-prefix argument is set. Security risk: enable this option only if a reverse proxy in front of thanos is resetting the header. The --web.prefix-header=X-Forwarded-Prefix option can be useful, for example, if Thanos UI is served via Traefik reverse proxy with PathPrefixStrip option enabled, which sends the stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path.").Default("").String() interval := cmd.Flag("refresh", "Refresh interval to download metadata from remote storage").Default("30m").Duration() timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() label := cmd.Flag("label", "Prometheus label to use as timeline title").String() - webExternalPrefix := cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the UI query web interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos UI to be served behind a reverse proxy that strips a URL sub-path.").Default("").String() m[name+" web"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { ctx, cancel := context.WithCancel(context.Background()) @@ -328,11 +329,13 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str flagsMap := map[string]string{ "web.external-prefix": *webExternalPrefix, + "web.prefix-header": *webPrefixHeaderName, } router := route.New() + bucketUI := ui.NewBucketUI(logger, *label, flagsMap) - bucketUI.Register(router, extpromhttp.NewInstrumentationMiddleware(reg)) + bucketUI.Register(router.WithPrefix(*webExternalPrefix), extpromhttp.NewInstrumentationMiddleware(reg)) srv.Handle("/", router) if *interval < 5*time.Minute { diff --git a/docs/components/bucket.md b/docs/components/bucket.md index fb977744b0..b6fc2c9188 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -126,16 +126,29 @@ Flags: Listen host:port for HTTP endpoints. --http-grace-period=2m Time to wait after an interrupt received for HTTP Server. + --web.external-prefix="" Static prefix for all HTML links and redirect + URLs in the bucket web UI interface. Actual + endpoints are still served on / or the + web.route-prefix. This allows thanos bucket web + UI to be served behind a reverse proxy that + strips a URL sub-path. + --web.prefix-header="" Name of HTTP request header used for dynamic + prefixing of UI links and redirects. This option + is ignored if web.external-prefix argument is + set. Security risk: enable this option only if a + reverse proxy in front of thanos is resetting + the header. The + --web.prefix-header=X-Forwarded-Prefix option + can be useful, for example, if Thanos UI is + served via Traefik reverse proxy with + PathPrefixStrip option enabled, which sends the + stripped prefix value in X-Forwarded-Prefix + header. This allows thanos UI to be served on a + sub-path. --refresh=30m Refresh interval to download metadata from remote storage --timeout=5m Timeout to download metadata from remote storage --label=LABEL Prometheus label to use as timeline title - --web.external-prefix="" Static prefix for all HTML links and redirect - URLs in the UI query web interface. Actual - endpoints are still served on / or the - web.route-prefix. This allows thanos UI to be - served behind a reverse proxy that strips a URL - sub-path. ``` diff --git a/pkg/ui/bucket.go b/pkg/ui/bucket.go index ae9fb2ad41..e8fcbe6887 100644 --- a/pkg/ui/bucket.go +++ b/pkg/ui/bucket.go @@ -13,13 +13,13 @@ import ( // Bucket is a web UI representing state of buckets as a timeline. type Bucket struct { *BaseUI + flagsMap map[string]string // Unique Prometheus label that identifies each shard, used as the title. If // not present, all labels are displayed externally as a legend. Label string Blocks template.JS RefreshedAt time.Time Err error - flagsMap map[string]string } func NewBucketUI(logger log.Logger, label string, flagsMap map[string]string) *Bucket { From 40cb6345ee23a21ee80d12ce3be99d58da7604a5 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 22 Nov 2019 18:10:43 +0100 Subject: [PATCH 058/257] ruler: fix the /api/v1/rules endpoint (#1773) * test/e2e: test alerts and rules API endpoints Signed-off-by: Simon Pasquier * pkg/rule: fix /api/v1/rules endpoint Signed-off-by: Simon Pasquier * Address Bartek's comments Signed-off-by: Simon Pasquier --- CHANGELOG.md | 1 + pkg/rule/api/v1.go | 6 +- pkg/rule/api/v1_test.go | 119 +++++++++++++++++++++------------------- pkg/rule/rule.go | 25 ++++----- test/e2e/rule_test.go | 46 +++++++++++----- 5 files changed, 110 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c24b3c09a8..5ca93d7d41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1669](https://github.com/thanos-io/thanos/pull/1669) Fixed store sharding. Now it does not load excluded meta.jsons and load/fetch index-cache.json files. - [#1670](https://github.com/thanos-io/thanos/pull/1670) Fixed un-ordered blocks upload. Sidecar now uploads the oldest blocks first. - [#1568](https://github.com/thanos-io/thanos/pull/1709) Thanos Store now retains the first raw value of a chunk during downsampling to avoid losing some counter resets that occur on an aggregation boundary. +- [#1773](https://github.com/thanos-io/thanos/pull/1773) Thanos Ruler: fixed the /api/v1/rules endpoint that returned 500 status code with `failed to assert type of rule ...` message. - [#1770](https://github.com/thanos-io/thanos/pull/1770) Fix `--web.external-prefix` 404s for static resources. ### Changed diff --git a/pkg/rule/api/v1.go b/pkg/rule/api/v1.go index 9b6d41a030..f49eb72291 100644 --- a/pkg/rule/api/v1.go +++ b/pkg/rule/api/v1.go @@ -84,7 +84,7 @@ func (api *API) rules(r *http.Request) (interface{}, []error, *qapi.ApiError) { } switch rule := r.(type) { - case thanosrule.AlertingRule: + case *rules.AlertingRule: enrichedRule = alertingRule{ Name: rule.Name(), Query: rule.Query().String(), @@ -95,7 +95,7 @@ func (api *API) rules(r *http.Request) (interface{}, []error, *qapi.ApiError) { Health: rule.Health(), LastError: lastError, Type: "alerting", - PartialResponseStrategy: rule.PartialResponseStrategy.String(), + PartialResponseStrategy: grp.PartialResponseStrategy.String(), } case *rules.RecordingRule: enrichedRule = recordingRule{ @@ -107,7 +107,7 @@ func (api *API) rules(r *http.Request) (interface{}, []error, *qapi.ApiError) { Type: "recording", } default: - err := fmt.Errorf("failed to assert type of rule '%v'", rule.Name()) + err := fmt.Errorf("rule %q: unsupported type %T", r.Name(), rule) return nil, nil, &qapi.ApiError{Typ: qapi.ErrorInternal, Err: err} } diff --git a/pkg/rule/api/v1_test.go b/pkg/rule/api/v1_test.go index 96a3a348fb..d949441740 100644 --- a/pkg/rule/api/v1_test.go +++ b/pkg/rule/api/v1_test.go @@ -62,8 +62,6 @@ type rulesRetrieverMock struct { } func (m rulesRetrieverMock) RuleGroups() []thanosrule.Group { - var ar rulesRetrieverMock - arules := ar.AlertingRules() storage := newStorage(m.testing) engineOpts := promql.EngineOpts{ @@ -83,9 +81,8 @@ func (m rulesRetrieverMock) RuleGroups() []thanosrule.Group { } var r []rules.Rule - - for _, alertrule := range arules { - r = append(r, alertrule) + for _, ar := range alertingRules(m.testing) { + r = append(r, ar) } recordingExpr, err := promql.ParseExpr(`vector(1)`) @@ -96,43 +93,49 @@ func (m rulesRetrieverMock) RuleGroups() []thanosrule.Group { r = append(r, recordingRule) group := rules.NewGroup("grp", "/path/to/file", time.Second, r, false, opts) - return []thanosrule.Group{{Group: group}} + return []thanosrule.Group{thanosrule.Group{Group: group}} } func (m rulesRetrieverMock) AlertingRules() []thanosrule.AlertingRule { + var ars []thanosrule.AlertingRule + for _, ar := range alertingRules(m.testing) { + ars = append(ars, thanosrule.AlertingRule{AlertingRule: ar}) + } + return ars +} + +func alertingRules(t *testing.T) []*rules.AlertingRule { expr1, err := promql.ParseExpr(`absent(test_metric3) != 1`) if err != nil { - m.testing.Fatalf("unable to parse alert expression: %s", err) + t.Fatalf("unable to parse alert expression: %s", err) } expr2, err := promql.ParseExpr(`up == 1`) if err != nil { - m.testing.Fatalf("Unable to parse alert expression: %s", err) + t.Fatalf("unable to parse alert expression: %s", err) } - rule1 := rules.NewAlertingRule( - "test_metric3", - expr1, - time.Second, - labels.Labels{}, - labels.Labels{}, - labels.Labels{}, - true, - log.NewNopLogger(), - ) - rule2 := rules.NewAlertingRule( - "test_metric4", - expr2, - time.Second, - labels.Labels{}, - labels.Labels{}, - labels.Labels{}, - true, - log.NewNopLogger(), - ) - var r []thanosrule.AlertingRule - r = append(r, thanosrule.AlertingRule{AlertingRule: rule1}) - r = append(r, thanosrule.AlertingRule{AlertingRule: rule2}) - return r + return []*rules.AlertingRule{ + rules.NewAlertingRule( + "test_metric3", + expr1, + time.Second, + labels.Labels{}, + labels.Labels{}, + labels.Labels{}, + true, + log.NewNopLogger(), + ), + rules.NewAlertingRule( + "test_metric4", + expr2, + time.Second, + labels.Labels{}, + labels.Labels{}, + labels.Labels{}, + true, + log.NewNopLogger(), + ), + } } func TestEndpoints(t *testing.T) { @@ -171,16 +174,17 @@ func TestEndpoints(t *testing.T) { } func testEndpoints(t *testing.T, api *API) { - type test struct { - endpoint qapi.ApiFunc - params map[string]string - query url.Values - response interface{} + endpointFn qapi.ApiFunc + endpointName string + params map[string]string + query url.Values + response interface{} } var tests = []test{ { - endpoint: api.rules, + endpointFn: api.rules, + endpointName: "rules", response: &RuleDiscovery{ RuleGroups: []*RuleGroup{ { @@ -232,27 +236,28 @@ func testEndpoints(t *testing.T, api *API) { request := func(m string, q url.Values) (*http.Request, error) { return http.NewRequest(m, fmt.Sprintf("http://example.com?%s", q.Encode()), nil) } - for i, test := range tests { - for _, method := range methods(test.endpoint) { - // Build a context with the correct request params. - ctx := context.Background() - for p, v := range test.params { - ctx = route.WithParam(ctx, p, v) - } - t.Logf("run %d\t%s\t%q", i, method, test.query.Encode()) + for _, test := range tests { + for _, method := range methods(test.endpointFn) { + t.Run(fmt.Sprintf("endpoint=%s/method=%s/query=%q", test.endpointName, method, test.query.Encode()), func(t *testing.T) { + // Build a context with the correct request params. + ctx := context.Background() + for p, v := range test.params { + ctx = route.WithParam(ctx, p, v) + } - req, err := request(method, test.query) - if err != nil { - t.Fatal(err) - } - endpoint, errors, apiError := test.endpoint(req.WithContext(ctx)) + req, err := request(method, test.query) + if err != nil { + t.Fatal(err) + } + endpoint, errors, apiError := test.endpointFn(req.WithContext(ctx)) - if errors != nil { - t.Fatalf("Unexpected errors: %s", errors) - return - } - assertAPIError(t, apiError) - assertAPIResponse(t, endpoint, test.response) + if errors != nil { + t.Fatalf("Unexpected errors: %s", errors) + return + } + assertAPIError(t, apiError) + assertAPIResponse(t, endpoint, test.response) + }) } } } diff --git a/pkg/rule/rule.go b/pkg/rule/rule.go index 5156d24d91..5bad87e4c1 100644 --- a/pkg/rule/rule.go +++ b/pkg/rule/rule.go @@ -112,8 +112,8 @@ func (r RuleGroup) MarshalYAML() (interface{}, error) { // special field in RuleGroup file. func (m *Managers) Update(dataDir string, evalInterval time.Duration, files []string) error { var ( - errs = tsdberrors.MultiError{} - filesMap = map[storepb.PartialResponseStrategy][]string{} + errs = tsdberrors.MultiError{} + filesByStrategy = map[storepb.PartialResponseStrategy][]string{} ) if err := os.RemoveAll(path.Join(dataDir, tmpRuleDir)); err != nil { @@ -138,19 +138,19 @@ func (m *Managers) Update(dataDir string, evalInterval time.Duration, files []st // NOTE: This is very ugly, but we need to reparse it into tmp dir without the field to have to reuse // rules.Manager. The problem is that it uses yaml.UnmarshalStrict for some reasons. - mapped := map[storepb.PartialResponseStrategy]*rulefmt.RuleGroups{} + groupsByStrategy := map[storepb.PartialResponseStrategy]*rulefmt.RuleGroups{} for _, rg := range rg.Groups { - if _, ok := mapped[*rg.PartialResponseStrategy]; !ok { - mapped[*rg.PartialResponseStrategy] = &rulefmt.RuleGroups{} + if _, ok := groupsByStrategy[*rg.PartialResponseStrategy]; !ok { + groupsByStrategy[*rg.PartialResponseStrategy] = &rulefmt.RuleGroups{} } - mapped[*rg.PartialResponseStrategy].Groups = append( - mapped[*rg.PartialResponseStrategy].Groups, + groupsByStrategy[*rg.PartialResponseStrategy].Groups = append( + groupsByStrategy[*rg.PartialResponseStrategy].Groups, rg.RuleGroup, ) } - for s, rg := range mapped { + for s, rg := range groupsByStrategy { b, err := yaml.Marshal(rg) if err != nil { errs = append(errs, err) @@ -163,20 +163,19 @@ func (m *Managers) Update(dataDir string, evalInterval time.Duration, files []st continue } - filesMap[s] = append(filesMap[s], newFn) + filesByStrategy[s] = append(filesByStrategy[s], newFn) } - } - for s, fs := range filesMap { - updater, ok := (*m)[s] + for s, fs := range filesByStrategy { + mgr, ok := (*m)[s] if !ok { errs = append(errs, errors.Errorf("no updater found for %v", s)) continue } // We add external labels in `pkg/alert.Queue`. // TODO(bwplotka): Investigate if we should put ext labels here or not. - if err := updater.Update(evalInterval, fs, nil); err != nil { + if err := mgr.Update(evalInterval, fs, nil); err != nil { errs = append(errs, err) continue } diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index d8c840c82c..272a3ed98a 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -189,7 +189,7 @@ func TestRule(t *testing.T) { return nil })) - // checks counter ensures we are not missing metrics. + // The checks counter ensures that we are not missing metrics. checks := 0 // Check metrics to make sure we report correct ones that allow handling the AlwaysFiring not being triggered because of query issue. testutil.Ok(t, promclient.MetricValues(ctx, nil, urlParse(t, r1.HTTP.URL()), func(lset labels.Labels, val float64) error { @@ -204,6 +204,15 @@ func TestRule(t *testing.T) { return nil })) testutil.Equals(t, 2, checks) + + // Verify API endpoints. + for _, endpoint := range []string{"/api/v1/rules", "/api/v1/alerts"} { + for _, r := range []*serverScheduler{r1, r2} { + code, _, err := getAPIEndpoint(ctx, r.HTTP.URL()+endpoint) + testutil.Ok(t, err) + testutil.Equals(t, 200, code) + } + } } type failingStoreAPI struct{} @@ -438,27 +447,17 @@ func TestRulePartialResponse(t *testing.T) { // TODO(bwplotka): Move to promclient. func queryAlertmanagerAlerts(ctx context.Context, url string) ([]*model.Alert, error) { - req, err := http.NewRequest("GET", url+"/api/v1/alerts", nil) + code, body, err := getAPIEndpoint(ctx, url+"/api/v1/alerts") if err != nil { return nil, err } - req = req.WithContext(ctx) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err + if code != 200 { + return nil, errors.Errorf("expected 200 response, got %d", code) } - defer runutil.CloseWithLogOnErr(nil, resp.Body, "close body query alertmanager") var v struct { Data []*model.Alert `json:"data"` } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if err = json.Unmarshal(body, &v); err != nil { return nil, err } @@ -468,3 +467,22 @@ func queryAlertmanagerAlerts(ctx context.Context, url string) ([]*model.Alert, e }) return v.Data, nil } + +func getAPIEndpoint(ctx context.Context, url string) (int, []byte, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return 0, nil, err + } + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, nil, err + } + defer runutil.CloseWithLogOnErr(nil, resp.Body, "%s: close body", req.URL.String()) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, nil, err + } + return resp.StatusCode, body, nil +} From f1ca3c1c22f270f8fe0c5bec7d55dc6d8d276594 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 22 Nov 2019 09:42:26 -0800 Subject: [PATCH 059/257] Moved 0.9.0 release to 26th of Nov (#1782) Due to KubeCon we need to delay the release to the next week ): It's already end of the week for most of us. Signed-off-by: Bartek Plotka --- docs/release-process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-process.md b/docs/release-process.md index dc61ac1e33..49c319c651 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -31,7 +31,7 @@ Release shepherd responsibilities: | Release | Time of first RC | Shepherd (Github handle) | |-----------|--------------------------|--------------------------| | v0.10.0 | (planned) 8.01.2019 | TBD | -| v0.9.0 | (planned) 21.11.2019 | `@bwplotka` | +| v0.9.0 | (planned) 26.11.2019 | `@bwplotka` | | v0.8.0 | (planned) 9.10.2019 | `@bwplotka` | | v0.7.0 | (planned) 28.08.2019 | `@domgreen` | | v0.6.0 | (planned) 12.07.2019 | `@GiedriusS` | From 6465838c5887d4e098ac2a2079e4fe5d7b9786fb Mon Sep 17 00:00:00 2001 From: Matt Layher Date: Sat, 23 Nov 2019 15:34:48 -0600 Subject: [PATCH 060/257] pkg: fix staticcheck issues (#1784) Signed-off-by: Matt Layher --- pkg/model/timeduration_test.go | 2 +- pkg/objstore/oss/oss.go | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/model/timeduration_test.go b/pkg/model/timeduration_test.go index d3d3eba271..30f2f71495 100644 --- a/pkg/model/timeduration_test.go +++ b/pkg/model/timeduration_test.go @@ -32,5 +32,5 @@ func TestTimeOrDurationValue(t *testing.T) { testutil.Assert(t, minTime.PrometheusTimestamp() > prevTime, "minTime prometheus timestamp is less than time now.") testutil.Assert(t, minTime.PrometheusTimestamp() < afterTime, "minTime prometheus timestamp is more than time now + 15s") - testutil.Assert(t, 253402300799000 == maxTime.PrometheusTimestamp(), "maxTime is not equal to 253402300799000") + testutil.Assert(t, maxTime.PrometheusTimestamp() == 253402300799000, "maxTime is not equal to 253402300799000") } diff --git a/pkg/objstore/oss/oss.go b/pkg/objstore/oss/oss.go index 95fe147b45..6f7b6320d3 100644 --- a/pkg/objstore/oss/oss.go +++ b/pkg/objstore/oss/oss.go @@ -63,15 +63,13 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { } func calculateChunks(name string, r io.Reader) (int, int64, error) { - switch r.(type) { + switch f := r.(type) { case *os.File: - f, _ := r.(*os.File) if fileInfo, err := f.Stat(); err == nil { s := fileInfo.Size() return int(math.Floor(float64(s) / PartSize)), s % PartSize, nil } case *strings.Reader: - f, _ := r.(*strings.Reader) return int(math.Floor(float64(f.Size()) / PartSize)), f.Size() % PartSize, nil } return -1, 0, errors.New("unsupported implement of io.Reader") From bacbb785b0da1db902e4d378ab4d86c0371f5d6e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 25 Nov 2019 01:55:37 +0000 Subject: [PATCH 061/257] build(deps): bump cloud.google.com/go from 0.48.0 to 0.49.0 (#1764) Bumps [cloud.google.com/go](https://github.com/GoogleCloudPlatform/gcloud-golang) from 0.48.0 to 0.49.0. - [Release notes](https://github.com/GoogleCloudPlatform/gcloud-golang/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/master/CHANGES.md) - [Commits](https://github.com/GoogleCloudPlatform/gcloud-golang/compare/v0.48.0...v0.49.0) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 17c51658cf..55a8b9c175 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/thanos-io/thanos require ( - cloud.google.com/go v0.48.0 + cloud.google.com/go v0.49.0 cloud.google.com/go/bigquery v1.3.0 // indirect cloud.google.com/go/storage v1.3.0 github.com/Azure/azure-pipeline-go v0.2.2 // indirect diff --git a/go.sum b/go.sum index 598db171d8..dbbe98e074 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.48.0 h1:6ZHYIRlohUdU4LrLHbTsReY1eYy/MoZW1FsEyBuMXsk= cloud.google.com/go v0.48.0/go.mod h1:gGOnoa/XMQYHAscREBlbdHduGchEaP9N0//OXdrPI/M= +cloud.google.com/go v0.49.0 h1:CH+lkubJzcPYB1Ggupcq0+k8Ni2ILdG2lYjDIgavDBQ= +cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= @@ -46,6 +48,7 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -282,6 +285,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -491,6 +495,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -500,6 +505,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -583,6 +589,8 @@ golang.org/x/tools v0.0.0-20191111182352-50fa39b762bc/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191114222411-4191b8cbba09/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -591,6 +599,7 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -610,6 +619,7 @@ google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBr google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de h1:dFEMUWudT9iV1JMk6i6NwbfIw2V/2VDFyDYCZFypRxE= google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -651,6 +661,7 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= From a4241a9fbad2f75b838ee24e45d8633dd29a2eb8 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Mon, 25 Nov 2019 16:41:24 +0100 Subject: [PATCH 062/257] test/e2e: start Alertmanager without clustering (#1786) Signed-off-by: Simon Pasquier --- test/e2e/spinup_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/spinup_test.go b/test/e2e/spinup_test.go index 9c6b3b09ad..e5f8979657 100644 --- a/test/e2e/spinup_test.go +++ b/test/e2e/spinup_test.go @@ -298,6 +298,7 @@ receivers: return newCmdExec(exec.Command(testutil.AlertmanagerBinary(), "--config.file", dir+"/config.yaml", "--web.listen-address", http.HostPort(), + "--cluster.listen-address", "", "--log.level", "debug", )), nil }, From e5d96c86f98ef334b2c333d57d7de89394061028 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Mon, 25 Nov 2019 19:18:39 +0100 Subject: [PATCH 063/257] pkg/rule: retain original path for rule files (#1785) This change ensures that the /api/v1/rules endpoint returns the original path of the rule file instead of the path of the temporary file generated by Thanos ruler. Signed-off-by: Simon Pasquier --- CHANGELOG.md | 1 + cmd/thanos/rule.go | 21 +++++----- pkg/rule/api/v1.go | 2 +- pkg/rule/api/v1_test.go | 14 +++++-- pkg/rule/rule.go | 78 +++++++++++++++++++++++++----------- pkg/rule/rule_test.go | 88 +++++++++++++++++++++++++---------------- pkg/ui/rule.go | 23 ++++++----- test/e2e/rule_test.go | 74 ++++++++++++++++++++++++++-------- test/e2e/spinup_test.go | 54 +++++++++---------------- 9 files changed, 218 insertions(+), 137 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ca93d7d41..74a5ac16e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1568](https://github.com/thanos-io/thanos/pull/1709) Thanos Store now retains the first raw value of a chunk during downsampling to avoid losing some counter resets that occur on an aggregation boundary. - [#1773](https://github.com/thanos-io/thanos/pull/1773) Thanos Ruler: fixed the /api/v1/rules endpoint that returned 500 status code with `failed to assert type of rule ...` message. - [#1770](https://github.com/thanos-io/thanos/pull/1770) Fix `--web.external-prefix` 404s for static resources. +- [#1785](https://github.com/thanos-io/thanos/pull/1785) Thanos Ruler: the /api/v1/rules endpoints now returns the original rule filenames. ### Changed diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index b34016376f..9660dbd275 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -291,7 +291,7 @@ func runRule( var ( alertmgrs = newAlertmanagerSet(logger, alertmgrURLs, dns.ResolverType(dnsSDResolver)) alertQ = alert.NewQueue(logger, reg, 10000, 100, labelsTSDBToProm(lset), alertExcludeLabels) - ruleMgrs = thanosrule.Managers{} + ruleMgr = thanosrule.NewManager(dataDir) ) { notify := func(ctx context.Context, expr string, alerts ...*rules.Alert) { @@ -338,15 +338,16 @@ func runRule( opts.Context = ctx opts.QueryFunc = queryFunc(logger, dnsProvider, duplicatedQuery, ruleEvalWarnings, s) - ruleMgrs[s] = rules.NewManager(&opts) + mgr := rules.NewManager(&opts) + ruleMgr.SetRuleManager(s, mgr) g.Add(func() error { - ruleMgrs[s].Run() + mgr.Run() <-ctx.Done() return nil }, func(error) { cancel() - ruleMgrs[s].Stop() + mgr.Stop() }) } } @@ -447,7 +448,7 @@ func runRule( level.Info(logger).Log("msg", "reload rule files", "numFiles", len(files)) - if err := ruleMgrs.Update(dataDir, evalInterval, files); err != nil { + if err := ruleMgr.Update(evalInterval, files); err != nil { configSuccess.Set(0) level.Error(logger).Log("msg", "reloading rules failed", "err", err) continue @@ -457,10 +458,8 @@ func runRule( configSuccessTime.Set(float64(time.Now().UnixNano()) / 1e9) rulesLoaded.Reset() - for s, mgr := range ruleMgrs { - for _, group := range mgr.RuleGroups() { - rulesLoaded.WithLabelValues(s.String(), group.File(), group.Name()).Set(float64(len(group.Rules()))) - } + for _, group := range ruleMgr.RuleGroups() { + rulesLoaded.WithLabelValues(group.PartialResponseStrategy.String(), group.File(), group.Name()).Set(float64(len(group.Rules()))) } } @@ -547,9 +546,9 @@ func runRule( ins := extpromhttp.NewInstrumentationMiddleware(reg) - ui.NewRuleUI(logger, reg, ruleMgrs, alertQueryURL.String(), flagsMap).Register(router.WithPrefix(webRoutePrefix), ins) + ui.NewRuleUI(logger, reg, ruleMgr, alertQueryURL.String(), flagsMap).Register(router.WithPrefix(webRoutePrefix), ins) - api := v1.NewAPI(logger, reg, ruleMgrs) + api := v1.NewAPI(logger, reg, ruleMgr) api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. diff --git a/pkg/rule/api/v1.go b/pkg/rule/api/v1.go index f49eb72291..bacdb5570a 100644 --- a/pkg/rule/api/v1.go +++ b/pkg/rule/api/v1.go @@ -69,7 +69,7 @@ func (api *API) rules(r *http.Request) (interface{}, []error, *qapi.ApiError) { for _, grp := range api.ruleRetriever.RuleGroups() { apiRuleGroup := &RuleGroup{ Name: grp.Name(), - File: grp.File(), + File: grp.OriginalFile(), Interval: grp.Interval().Seconds(), Rules: []rule{}, PartialResponseStrategy: grp.PartialResponseStrategy.String(), diff --git a/pkg/rule/api/v1_test.go b/pkg/rule/api/v1_test.go index d949441740..ce71f5d16b 100644 --- a/pkg/rule/api/v1_test.go +++ b/pkg/rule/api/v1_test.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/prometheus/storage/tsdb" qapi "github.com/thanos-io/thanos/pkg/query/api" thanosrule "github.com/thanos-io/thanos/pkg/rule" + "github.com/thanos-io/thanos/pkg/store/storepb" ) // NewStorage returns a new storage for testing purposes @@ -92,8 +93,12 @@ func (m rulesRetrieverMock) RuleGroups() []thanosrule.Group { recordingRule := rules.NewRecordingRule("recording-rule-1", recordingExpr, labels.Labels{}) r = append(r, recordingRule) - group := rules.NewGroup("grp", "/path/to/file", time.Second, r, false, opts) - return []thanosrule.Group{thanosrule.Group{Group: group}} + return []thanosrule.Group{ + thanosrule.Group{ + Group: rules.NewGroup("grp", "/path/to/file", time.Second, r, false, opts), + PartialResponseStrategy: storepb.PartialResponseStrategy_WARN, + }, + } } func (m rulesRetrieverMock) AlertingRules() []thanosrule.AlertingRule { @@ -189,7 +194,7 @@ func testEndpoints(t *testing.T, api *API) { RuleGroups: []*RuleGroup{ { Name: "grp", - File: "/path/to/file", + File: "", Interval: 1, PartialResponseStrategy: "WARN", Rules: []rule{ @@ -263,13 +268,14 @@ func testEndpoints(t *testing.T, api *API) { } func assertAPIError(t *testing.T, got *qapi.ApiError) { + t.Helper() if got != nil { t.Fatalf("Unexpected error: %s", got) - return } } func assertAPIResponse(t *testing.T, got interface{}, exp interface{}) { + t.Helper() if !reflect.DeepEqual(exp, got) { respJSON, err := json.Marshal(got) if err != nil { diff --git a/pkg/rule/rule.go b/pkg/rule/rule.go index 5bad87e4c1..970ffbe8bc 100644 --- a/pkg/rule/rule.go +++ b/pkg/rule/rule.go @@ -4,9 +4,9 @@ import ( "fmt" "io/ioutil" "os" - "path" "path/filepath" "strings" + "sync" "time" "github.com/pkg/errors" @@ -21,9 +21,14 @@ const tmpRuleDir = ".tmp-rules" type Group struct { *rules.Group + originalFile string PartialResponseStrategy storepb.PartialResponseStrategy } +func (g Group) OriginalFile() string { + return g.originalFile +} + type AlertingRule struct { *rules.AlertingRule PartialResponseStrategy storepb.PartialResponseStrategy @@ -38,21 +43,45 @@ type RuleGroup struct { PartialResponseStrategy *storepb.PartialResponseStrategy } -type Managers map[storepb.PartialResponseStrategy]*rules.Manager +type Manager struct { + workDir string + mgrs map[storepb.PartialResponseStrategy]*rules.Manager + + mtx sync.RWMutex + ruleFiles map[string]string +} -func (m Managers) RuleGroups() []Group { +func NewManager(dataDir string) *Manager { + return &Manager{ + workDir: filepath.Join(dataDir, tmpRuleDir), + mgrs: make(map[storepb.PartialResponseStrategy]*rules.Manager), + ruleFiles: make(map[string]string), + } +} + +func (m *Manager) SetRuleManager(s storepb.PartialResponseStrategy, mgr *rules.Manager) { + m.mgrs[s] = mgr +} + +func (m *Manager) RuleGroups() []Group { + m.mtx.RLock() + defer m.mtx.RUnlock() var res []Group - for s, r := range m { + for s, r := range m.mgrs { for _, group := range r.RuleGroups() { - res = append(res, Group{Group: group, PartialResponseStrategy: s}) + res = append(res, Group{ + Group: group, + PartialResponseStrategy: s, + originalFile: m.ruleFiles[group.File()], + }) } } return res } -func (m Managers) AlertingRules() []AlertingRule { +func (m *Manager) AlertingRules() []AlertingRule { var res []AlertingRule - for s, r := range m { + for s, r := range m.mgrs { for _, r := range r.AlertingRules() { res = append(res, AlertingRule{AlertingRule: r, PartialResponseStrategy: s}) } @@ -67,12 +96,12 @@ func (r *RuleGroup) UnmarshalYAML(unmarshal func(interface{}) error) error { errMsg := fmt.Sprintf("failed to unmarshal 'partial_response_strategy'. Possible values are %s", strings.Join(storepb.PartialResponseStrategyValues, ",")) if err := unmarshal(&rs); err != nil { - return errors.Wrapf(err, errMsg) + return errors.Wrap(err, errMsg) } rg := rulefmt.RuleGroup{} if err := unmarshal(&rg); err != nil { - return errors.Wrapf(err, errMsg) + return errors.Wrap(err, "failed to unmarshal rulefmt.RuleGroup") } p, ok := storepb.PartialResponseStrategy_value[strings.ToUpper(rs.String)] @@ -110,17 +139,18 @@ func (r RuleGroup) MarshalYAML() (interface{}, error) { // Update updates rules from given files to all managers we hold. We decide which groups should go where, based on // special field in RuleGroup file. -func (m *Managers) Update(dataDir string, evalInterval time.Duration, files []string) error { +func (m *Manager) Update(evalInterval time.Duration, files []string) error { var ( - errs = tsdberrors.MultiError{} + errs tsdberrors.MultiError filesByStrategy = map[storepb.PartialResponseStrategy][]string{} + ruleFiles = map[string]string{} ) - if err := os.RemoveAll(path.Join(dataDir, tmpRuleDir)); err != nil { - return errors.Wrapf(err, "rm %s", path.Join(dataDir, tmpRuleDir)) + if err := os.RemoveAll(m.workDir); err != nil { + return errors.Wrapf(err, "failed to remove %s", m.workDir) } - if err := os.MkdirAll(path.Join(dataDir, tmpRuleDir), os.ModePerm); err != nil { - return errors.Wrapf(err, "mkdir %s", path.Join(dataDir, tmpRuleDir)) + if err := os.MkdirAll(m.workDir, os.ModePerm); err != nil { + return errors.Wrapf(err, "failed to create %s", m.workDir) } for _, fn := range files { @@ -132,7 +162,7 @@ func (m *Managers) Update(dataDir string, evalInterval time.Duration, files []st var rg RuleGroups if err := yaml.Unmarshal(b, &rg); err != nil { - errs = append(errs, err) + errs = append(errs, errors.Wrap(err, fn)) continue } @@ -153,33 +183,37 @@ func (m *Managers) Update(dataDir string, evalInterval time.Duration, files []st for s, rg := range groupsByStrategy { b, err := yaml.Marshal(rg) if err != nil { - errs = append(errs, err) + errs = append(errs, errors.Wrapf(err, "%s: failed to marshal rule groups", fn)) continue } - newFn := path.Join(dataDir, tmpRuleDir, filepath.Base(fn)+"."+s.String()) + newFn := filepath.Join(m.workDir, filepath.Base(fn)+"."+s.String()) if err := ioutil.WriteFile(newFn, b, os.ModePerm); err != nil { - errs = append(errs, err) + errs = append(errs, errors.Wrap(err, newFn)) continue } filesByStrategy[s] = append(filesByStrategy[s], newFn) + ruleFiles[newFn] = fn } } + m.mtx.Lock() for s, fs := range filesByStrategy { - mgr, ok := (*m)[s] + mgr, ok := m.mgrs[s] if !ok { - errs = append(errs, errors.Errorf("no updater found for %v", s)) + errs = append(errs, errors.Errorf("no manager found for %v", s)) continue } // We add external labels in `pkg/alert.Queue`. // TODO(bwplotka): Investigate if we should put ext labels here or not. if err := mgr.Update(evalInterval, fs, nil); err != nil { - errs = append(errs, err) + errs = append(errs, errors.Wrapf(err, "strategy %s", s)) continue } } + m.ruleFiles = ruleFiles + m.mtx.Unlock() return errs.Err() } diff --git a/pkg/rule/rule_test.go b/pkg/rule/rule_test.go index 5353bb3503..8b9864a17e 100644 --- a/pkg/rule/rule_test.go +++ b/pkg/rule/rule_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "sort" "strings" "testing" @@ -70,59 +71,76 @@ groups: - alert: "some" expr: "up" `), os.ModePerm)) - testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "combined-wrong.yaml"), []byte(` -groups: -- name: "something8" - partial_response_strategy: "warn" - rules: - - alert: "some" - expr: "up" -- name: "something9" - partial_response_strategy: "adad" # Err 2 - rules: - - alert: "some" - expr: "up" -`), os.ModePerm)) opts := rules.ManagerOptions{ Logger: log.NewLogfmtLogger(os.Stderr), } - m := Managers{ - storepb.PartialResponseStrategy_ABORT: rules.NewManager(&opts), - storepb.PartialResponseStrategy_WARN: rules.NewManager(&opts), - } + m := NewManager(dir) + m.SetRuleManager(storepb.PartialResponseStrategy_ABORT, rules.NewManager(&opts)) + m.SetRuleManager(storepb.PartialResponseStrategy_WARN, rules.NewManager(&opts)) - err = m.Update(dir, 10*time.Second, []string{ + err = m.Update(10*time.Second, []string{ path.Join(dir, "no_strategy.yaml"), path.Join(dir, "abort.yaml"), path.Join(dir, "warn.yaml"), path.Join(dir, "wrong.yaml"), path.Join(dir, "combined.yaml"), - path.Join(dir, "combined_wrong.yaml"), + path.Join(dir, "non_existing.yaml"), }) testutil.NotOk(t, err) - testutil.Assert(t, strings.HasPrefix(err.Error(), "2 errors: failed to unmarshal 'partial_response_strategy'"), err.Error()) - - g := m[storepb.PartialResponseStrategy_WARN].RuleGroups() - testutil.Equals(t, 2, len(g)) + testutil.Assert(t, strings.Contains(err.Error(), "wrong.yaml: failed to unmarshal 'partial_response_strategy'"), err.Error()) + testutil.Assert(t, strings.Contains(err.Error(), "non_existing.yaml: no such file or directory"), err.Error()) + g := m.RuleGroups() sort.Slice(g, func(i, j int) bool { return g[i].Name() < g[j].Name() }) - testutil.Equals(t, "something3", g[0].Name()) - testutil.Equals(t, "something5", g[1].Name()) - - g = m[storepb.PartialResponseStrategy_ABORT].RuleGroups() - testutil.Equals(t, 4, len(g)) - sort.Slice(g, func(i, j int) bool { - return g[i].Name() < g[j].Name() - }) - testutil.Equals(t, "something1", g[0].Name()) - testutil.Equals(t, "something2", g[1].Name()) - testutil.Equals(t, "something6", g[2].Name()) - testutil.Equals(t, "something7", g[3].Name()) + exp := []struct { + name string + file string + strategy storepb.PartialResponseStrategy + }{ + { + name: "something1", + file: filepath.Join(dir, "no_strategy.yaml"), + strategy: storepb.PartialResponseStrategy_ABORT, + }, + { + name: "something2", + file: filepath.Join(dir, "abort.yaml"), + strategy: storepb.PartialResponseStrategy_ABORT, + }, + { + name: "something3", + file: filepath.Join(dir, "warn.yaml"), + strategy: storepb.PartialResponseStrategy_WARN, + }, + { + name: "something5", + file: filepath.Join(dir, "combined.yaml"), + strategy: storepb.PartialResponseStrategy_WARN, + }, + { + name: "something6", + file: filepath.Join(dir, "combined.yaml"), + strategy: storepb.PartialResponseStrategy_ABORT, + }, + { + name: "something7", + file: filepath.Join(dir, "combined.yaml"), + strategy: storepb.PartialResponseStrategy_ABORT, + }, + } + testutil.Equals(t, len(exp), len(g)) + for i := range exp { + t.Run(exp[i].name, func(t *testing.T) { + testutil.Equals(t, exp[i].strategy, g[i].PartialResponseStrategy) + testutil.Equals(t, exp[i].name, g[i].Name()) + testutil.Equals(t, exp[i].file, g[i].OriginalFile()) + }) + } } func TestRuleGroupMarshalYAML(t *testing.T) { diff --git a/pkg/ui/rule.go b/pkg/ui/rule.go index 39b753d726..12188df9d9 100644 --- a/pkg/ui/rule.go +++ b/pkg/ui/rule.go @@ -16,7 +16,6 @@ import ( "github.com/prometheus/prometheus/rules" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" thanosrule "github.com/thanos-io/thanos/pkg/rule" - "github.com/thanos-io/thanos/pkg/store/storepb" ) type Rule struct { @@ -24,18 +23,18 @@ type Rule struct { flagsMap map[string]string - ruleManagers thanosrule.Managers - queryURL string - reg prometheus.Registerer + ruleManager *thanosrule.Manager + queryURL string + reg prometheus.Registerer } -func NewRuleUI(logger log.Logger, reg prometheus.Registerer, ruleManagers map[storepb.PartialResponseStrategy]*rules.Manager, queryURL string, flagsMap map[string]string) *Rule { +func NewRuleUI(logger log.Logger, reg prometheus.Registerer, ruleManager *thanosrule.Manager, queryURL string, flagsMap map[string]string) *Rule { return &Rule{ - BaseUI: NewBaseUI(logger, "rule_menu.html", ruleTmplFuncs(queryURL)), - flagsMap: flagsMap, - ruleManagers: ruleManagers, - queryURL: queryURL, - reg: reg, + BaseUI: NewBaseUI(logger, "rule_menu.html", ruleTmplFuncs(queryURL)), + flagsMap: flagsMap, + ruleManager: ruleManager, + queryURL: queryURL, + reg: reg, } } @@ -115,7 +114,7 @@ func ruleTmplFuncs(queryURL string) template.FuncMap { } func (ru *Rule) alerts(w http.ResponseWriter, r *http.Request) { - alerts := ru.ruleManagers.AlertingRules() + alerts := ru.ruleManager.AlertingRules() alertsSorter := byAlertStateAndNameSorter{alerts: alerts} sort.Sort(alertsSorter) @@ -138,7 +137,7 @@ func (ru *Rule) rules(w http.ResponseWriter, r *http.Request) { prefix := GetWebPrefix(ru.logger, ru.flagsMap, r) // TODO(bwplotka): Update HTML to include partial response. - ru.executeTemplate(w, "rules.html", prefix, ru.ruleManagers) + ru.executeTemplate(w, "rules.html", prefix, ru.ruleManager) } // Root redirects / requests to /graph, taking into account the path prefix value. diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 272a3ed98a..5784938aa8 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -8,7 +8,7 @@ import ( "math" "net/http" "os" - "path" + "path/filepath" "sort" "testing" "time" @@ -18,6 +18,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/thanos-io/thanos/pkg/promclient" + rapi "github.com/thanos-io/thanos/pkg/rule/api" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" @@ -52,9 +53,14 @@ groups: ` ) -var ( - alertsToTest = []string{testAlertRuleAbortOnPartialResponse, testAlertRuleWarnOnPartialResponse} -) +func createRuleFiles(t *testing.T, dir string) { + t.Helper() + + for i, rule := range []string{testAlertRuleAbortOnPartialResponse, testAlertRuleWarnOnPartialResponse} { + err := ioutil.WriteFile(filepath.Join(dir, fmt.Sprintf("rules-%d.yaml", i)), []byte(rule), 0666) + testutil.Ok(t, err) + } +} func TestRule(t *testing.T) { a := newLocalAddresser() @@ -62,8 +68,13 @@ func TestRule(t *testing.T) { am := alertManager(a.New()) qAddr := a.New() - r1 := rule(a.New(), a.New(), alertsToTest, am.HTTP, []address{qAddr}, nil) - r2 := rule(a.New(), a.New(), alertsToTest, am.HTTP, nil, []address{qAddr}) + rulesDir, err := ioutil.TempDir("", "rules") + defer os.RemoveAll(rulesDir) + testutil.Ok(t, err) + createRuleFiles(t, rulesDir) + + r1 := rule(a.New(), a.New(), rulesDir, am.HTTP, []address{qAddr}, nil) + r2 := rule(a.New(), a.New(), rulesDir, am.HTTP, nil, []address{qAddr}) q := querier(qAddr, a.New(), []address{r1.GRPC, r2.GRPC}, nil) @@ -205,14 +216,23 @@ func TestRule(t *testing.T) { })) testutil.Equals(t, 2, checks) - // Verify API endpoints. - for _, endpoint := range []string{"/api/v1/rules", "/api/v1/alerts"} { - for _, r := range []*serverScheduler{r1, r2} { - code, _, err := getAPIEndpoint(ctx, r.HTTP.URL()+endpoint) - testutil.Ok(t, err) - testutil.Equals(t, 200, code) + // Verify the rules API endpoint. + for _, r := range []*serverScheduler{r1, r2} { + rgs, err := queryRules(ctx, r.HTTP.URL()) + testutil.Ok(t, err) + testutil.Equals(t, 2, len(rgs)) + for i := range rgs { + testutil.Equals(t, filepath.Join(rulesDir, fmt.Sprintf("rules-%d.yaml", i)), rgs[i].File) + testutil.Equals(t, "example", rgs[i].Name) } } + + // Verify the alerts API endpoint. + for _, r := range []*serverScheduler{r1, r2} { + code, _, err := getAPIEndpoint(ctx, r.HTTP.URL()+"/api/v1/alerts") + testutil.Ok(t, err) + testutil.Equals(t, 200, code) + } } type failingStoreAPI struct{} @@ -271,7 +291,10 @@ func TestRulePartialResponse(t *testing.T) { f := fakeStoreAPI(a.New(), &failingStoreAPI{}) am := alertManager(a.New()) - r := ruleWithDir(a.New(), a.New(), dir, nil, am.HTTP, []address{qAddr}, nil) + rulesDir, err := ioutil.TempDir("", "rules") + defer os.RemoveAll(rulesDir) + testutil.Ok(t, err) + r := rule(a.New(), a.New(), rulesDir, am.HTTP, []address{qAddr}, nil) q := querier(qAddr, a.New(), []address{r.GRPC, f.GRPC}, nil) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) @@ -316,9 +339,7 @@ func TestRulePartialResponse(t *testing.T) { })) // Add alerts to ruler, we want to add it only when Querier is rdy, otherwise we will get "no store match the query". - for i, rule := range alertsToTest { - testutil.Ok(t, ioutil.WriteFile(path.Join(dir, fmt.Sprintf("rules-%d.yaml", i)), []byte(rule), 0666)) - } + createRuleFiles(t, rulesDir) resp, err := http.Post(r.HTTP.URL()+"/-/reload", "", nil) testutil.Ok(t, err) @@ -468,6 +489,27 @@ func queryAlertmanagerAlerts(ctx context.Context, url string) ([]*model.Alert, e return v.Data, nil } +func queryRules(ctx context.Context, url string) ([]*rapi.RuleGroup, error) { + code, body, err := getAPIEndpoint(ctx, url+"/api/v1/rules") + if err != nil { + return nil, err + } + if code != 200 { + return nil, errors.Errorf("expected 200 response, got %d", code) + } + + var resp struct { + Data rapi.RuleDiscovery + } + if err = json.Unmarshal(body, &resp); err != nil { + return nil, err + } + sort.Slice(resp.Data.RuleGroups, func(i, j int) bool { + return resp.Data.RuleGroups[i].File < resp.Data.RuleGroups[j].File + }) + return resp.Data.RuleGroups, nil +} + func getAPIEndpoint(ctx context.Context, url string) (int, []byte, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { diff --git a/test/e2e/spinup_test.go b/test/e2e/spinup_test.go index e5f8979657..4eea79e5fc 100644 --- a/test/e2e/spinup_test.go +++ b/test/e2e/spinup_test.go @@ -10,7 +10,7 @@ import ( "net" "os" "os/exec" - "path" + "path/filepath" "strconv" "syscall" "testing" @@ -120,13 +120,13 @@ type prometheusScheduler struct { func prometheus(http address, config string) *prometheusScheduler { s := &prometheusScheduler{ - RelDir: path.Join("data", "prom", http.Port), + RelDir: filepath.Join("data", "prom", http.Port), } s.serverScheduler = serverScheduler{ HTTP: http, schedule: func(workDir string) (execs Exec, e error) { - promDir := path.Join(workDir, s.RelDir) + promDir := filepath.Join(workDir, s.RelDir) if err := os.MkdirAll(promDir, 0777); err != nil { return nil, errors.Wrap(err, "create prom dir failed") } @@ -152,7 +152,7 @@ func sidecar(http, grpc address, prom *prometheusScheduler) *serverScheduler { HTTP: http, GRPC: grpc, schedule: func(workDir string) (Exec, error) { - promDir := path.Join(workDir, prom.RelDir) + promDir := filepath.Join(workDir, prom.RelDir) return newCmdExec(exec.Command("thanos", "sidecar", "--debug.name", fmt.Sprintf("sidecar-%s", http.Port), "--grpc-address", grpc.HostPort(), @@ -174,7 +174,7 @@ func receiver(http, grpc, metric address, replicationFactor int, hashring ...rec HTTP: http, GRPC: grpc, schedule: func(workDir string) (Exec, error) { - receiveDir := path.Join(workDir, "data", "receive", http.Port) + receiveDir := filepath.Join(workDir, "data", "receive", http.Port) if err := os.MkdirAll(receiveDir, 0777); err != nil { return nil, errors.Wrap(err, "create receive dir") } @@ -184,7 +184,7 @@ func receiver(http, grpc, metric address, replicationFactor int, hashring ...rec return nil, errors.Wrapf(err, "generate hashring file: %v", hashring) } - if err := ioutil.WriteFile(path.Join(receiveDir, "hashrings.json"), b, 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(receiveDir, "hashrings.json"), b, 0666); err != nil { return nil, errors.Wrap(err, "creating receive config") } @@ -195,11 +195,11 @@ func receiver(http, grpc, metric address, replicationFactor int, hashring ...rec "--http-address", metric.HostPort(), "--remote-write.address", http.HostPort(), "--label", fmt.Sprintf(`receive="%s"`, http.Port), - "--tsdb.path", path.Join(receiveDir, "tsdb"), + "--tsdb.path", filepath.Join(receiveDir, "tsdb"), "--log.level", "debug", "--receive.replication-factor", strconv.Itoa(replicationFactor), "--receive.local-endpoint", remoteWriteEndpoint(http), - "--receive.hashrings-file", path.Join(receiveDir, "hashrings.json"), + "--receive.hashrings-file", filepath.Join(receiveDir, "hashrings.json"), "--receive.hashrings-file-refresh-interval", "5s")), nil }, } @@ -226,7 +226,7 @@ func querier(http, grpc address, storeAddresses []address, fileSDStoreAddresses } if len(fileSDStoreAddresses) > 0 { - queryFileSDDir := path.Join(workDir, "data", "querier", http.Port) + queryFileSDDir := filepath.Join(workDir, "data", "querier", http.Port) if err := os.MkdirAll(queryFileSDDir, 0777); err != nil { return nil, errors.Wrap(err, "create query dir failed") } @@ -236,7 +236,7 @@ func querier(http, grpc address, storeAddresses []address, fileSDStoreAddresses } args = append(args, - "--store.sd-files", path.Join(queryFileSDDir, "filesd.json"), + "--store.sd-files", filepath.Join(queryFileSDDir, "filesd.json"), "--store.sd-interval", "5s", ) } @@ -251,7 +251,7 @@ func storeGateway(http, grpc address, bucketConfig []byte, relabelConfig []byte) HTTP: http, GRPC: grpc, schedule: func(workDir string) (Exec, error) { - dbDir := path.Join(workDir, "data", "store-gateway", http.Port) + dbDir := filepath.Join(workDir, "data", "store-gateway", http.Port) if err := os.MkdirAll(dbDir, 0777); err != nil { return nil, errors.Wrap(err, "creating store gateway dir failed") @@ -278,7 +278,7 @@ func alertManager(http address) *serverScheduler { return &serverScheduler{ HTTP: http, schedule: func(workDir string) (Exec, error) { - dir := path.Join(workDir, "data", "alertmanager", http.Port) + dir := filepath.Join(workDir, "data", "alertmanager", http.Port) if err := os.MkdirAll(dir, 0777); err != nil { return nil, errors.Wrap(err, "creating alertmanager dir failed") @@ -305,35 +305,17 @@ receivers: } } -func rule(http, grpc address, rules []string, am address, queryAddresses []address, queryFileSDAddresses []address) *serverScheduler { - return ruleWithDir(http, grpc, "", rules, am, queryAddresses, queryFileSDAddresses) -} - -func ruleWithDir(http, grpc address, dir string, rules []string, am address, queryAddresses []address, queryFileSDAddresses []address) *serverScheduler { +func rule(http, grpc address, ruleDir string, am address, queryAddresses []address, queryFileSDAddresses []address) *serverScheduler { return &serverScheduler{ HTTP: http, GRPC: grpc, schedule: func(workDir string) (Exec, error) { - ruleDir := path.Join(workDir, "data", "rule", http.Port) - if dir != "" { - ruleDir = dir - } - - if err := os.MkdirAll(ruleDir, 0777); err != nil { - return nil, errors.Wrap(err, "creating ruler dir") - } - for i, rule := range rules { - if err := ioutil.WriteFile(path.Join(ruleDir, fmt.Sprintf("/rules-%d.yaml", i)), []byte(rule), 0666); err != nil { - return nil, errors.Wrapf(err, "writing rule %s", path.Join(ruleDir, fmt.Sprintf("/rules-%d.yaml", i))) - } - } - args := []string{ "rule", "--debug.name", fmt.Sprintf("rule-%s", http.Port), "--label", fmt.Sprintf(`replica="%s"`, http.Port), - "--data-dir", path.Join(ruleDir, "data"), - "--rule-file", path.Join(ruleDir, "*.yaml"), + "--data-dir", filepath.Join(workDir, "data"), + "--rule-file", filepath.Join(ruleDir, "*.yaml"), "--eval-interval", "1s", "--alertmanagers.url", am.URL(), "--grpc-address", grpc.HostPort(), @@ -348,10 +330,10 @@ func ruleWithDir(http, grpc address, dir string, rules []string, am address, que } if len(queryFileSDAddresses) > 0 { - if err := ioutil.WriteFile(path.Join(ruleDir, "filesd.json"), []byte(generateFileSD(queryFileSDAddresses)), 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(workDir, "filesd.json"), []byte(generateFileSD(queryFileSDAddresses)), 0666); err != nil { return nil, errors.Wrap(err, "creating ruler filesd config") } - args = append(args, "--query.sd-files", path.Join(ruleDir, "filesd.json")) + args = append(args, "--query.sd-files", filepath.Join(workDir, "filesd.json")) } return newCmdExec(exec.Command("thanos", args...)), nil }, @@ -431,7 +413,7 @@ func minio(http address, config s3.Config) *serverScheduler { return &serverScheduler{ HTTP: http, schedule: func(workDir string) (Exec, error) { - dbDir := path.Join(workDir, "data", "minio", http.Port) + dbDir := filepath.Join(workDir, "data", "minio", http.Port) if err := os.MkdirAll(dbDir, 0777); err != nil { return nil, errors.Wrap(err, "creating minio dir failed") } From 825f119982dd2db4b9a3b4fe0a4b652331670ff2 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Tue, 26 Nov 2019 09:08:20 +0100 Subject: [PATCH 064/257] Simplify prober (#1694) * Simplify prober Signed-off-by: Kemal Akkoyun * Preserve existing behaviour Signed-off-by: Kemal Akkoyun * Add CHANGELOG entry Signed-off-by: Kemal Akkoyun --- CHANGELOG.md | 1 + cmd/thanos/bucket.go | 13 ++- cmd/thanos/compact.go | 15 +++- cmd/thanos/downsample.go | 15 +++- cmd/thanos/query.go | 19 ++-- cmd/thanos/receive.go | 17 +++- cmd/thanos/rule.go | 18 +++- cmd/thanos/sidecar.go | 21 +++-- cmd/thanos/store.go | 17 +++- pkg/prober/prober.go | 173 +++++++++++++++-------------------- pkg/prober/prober_test.go | 183 ++++++++++++++++---------------------- pkg/server/http/http.go | 17 ++-- 12 files changed, 262 insertions(+), 247 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a5ac16e3..0312950027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1666](https://github.com/thanos-io/thanos/pull/1666) `thanos_compact_group_compactions_total` now counts block compactions, so operations that resulted in a compacted block. The old behaviour is now exposed by new metric: `thanos_compact_group_compaction_runs_started_total` and `thanos_compact_group_compaction_runs_completed_total` which counts compaction runs overall. - [#1748](https://github.com/thanos-io/thanos/pull/1748) Updated all dependencies. +- [#1694](https://github.com/thanos-io/thanos/pull/1694) `prober_ready` and `prober_healthy` metrics are removed, for sake of `status`. Now `status` exposes same metric with a label, `check`. `check` can have "healty" or "ready" depending on status of the probe. ## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14 diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 0f7c43cc94..4d1e4e92ca 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -320,7 +320,7 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str m[name+" web"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { ctx, cancel := context.WithCancel(context.Background()) - statusProber := prober.NewProber(component.Bucket, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + statusProber := prober.New(component.Bucket, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. srv := httpserver.New(logger, reg, component.Bucket, statusProber, httpserver.WithListen(*httpBindAddr), @@ -356,7 +356,16 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str cancel() }) - g.Add(srv.ListenAndServe, srv.Shutdown) + g.Add(func() error { + statusProber.Healthy() + + return srv.ListenAndServe() + }, func(err error) { + statusProber.NotReady(err) + defer statusProber.NotHealthy(err) + + srv.Shutdown(err) + }) return nil } diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 2589a251ee..1a08135744 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -180,14 +180,23 @@ func runCompact( downsampleMetrics := newDownsampleMetrics(reg) - statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + statusProber := prober.New(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. srv := httpserver.New(logger, reg, component, statusProber, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) - g.Add(srv.ListenAndServe, srv.Shutdown) + g.Add(func() error { + statusProber.Healthy() + + return srv.ListenAndServe() + }, func(err error) { + statusProber.NotReady(err) + defer statusProber.NotHealthy(err) + + srv.Shutdown(err) + }) confContentYaml, err := objStoreConfig.Content() if err != nil { @@ -350,7 +359,7 @@ func runCompact( }) level.Info(logger).Log("msg", "starting compact node") - statusProber.SetReady() + statusProber.Ready() return nil } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 655bc20b63..5c053d7e6e 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -96,14 +96,14 @@ func runDownsample( }() metrics := newDownsampleMetrics(reg) - statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Start cycle of syncing blocks from the bucket and garbage collecting the bucket. { ctx, cancel := context.WithCancel(context.Background()) g.Add(func() error { defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") - statusProber.SetReady() + statusProber.Ready() level.Info(logger).Log("msg", "start first pass of downsampling") @@ -128,7 +128,16 @@ func runDownsample( httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) - g.Add(srv.ListenAndServe, srv.Shutdown) + g.Add(func() error { + statusProber.Healthy() + + return srv.ListenAndServe() + }, func(err error) { + statusProber.NotReady(err) + defer statusProber.NotHealthy(err) + + srv.Shutdown(err) + }) level.Info(logger).Log("msg", "starting downsample node") return nil diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 2c836a5b81..f75c23f45b 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -355,7 +355,7 @@ func runQuery( } // Start query API + UI HTTP server. - statusProber := prober.NewProber(comp, logger, reg) + statusProber := prober.New(comp, logger, reg) { router := route.New() @@ -386,7 +386,16 @@ func runQuery( ) srv.Handle("/", router) - g.Add(srv.ListenAndServe, srv.Shutdown) + g.Add(func() error { + statusProber.Healthy() + + return srv.ListenAndServe() + }, func(err error) { + statusProber.NotReady(err) + defer statusProber.NotHealthy(err) + + srv.Shutdown(err) + }) } // Start query (proxy) gRPC StoreAPI. { @@ -402,10 +411,10 @@ func runQuery( ) g.Add(func() error { - statusProber.SetReady() + statusProber.Ready() return s.ListenAndServe() - }, func(err error) { - statusProber.SetNotReady(err) + }, func(error) { + statusProber.NotReady(err) s.Shutdown(err) }) } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 18d4474066..5571dbd88b 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -198,7 +198,7 @@ func runReceive( TLSClientConfig: rwTLSClientConfig, }) - statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) confContentYaml, err := objStoreConfig.Content() if err != nil { return err @@ -268,7 +268,7 @@ func runReceive( level.Info(logger).Log("msg", "tsdb started") localStorage.Set(db.Get(), startTimeMargin) webHandler.SetWriter(receive.NewWriter(log.With(logger, "component", "receive-writer"), localStorage)) - statusProber.SetReady() + statusProber.Ready() level.Info(logger).Log("msg", "server is ready to receive web requests.") dbReady <- struct{}{} } @@ -318,7 +318,7 @@ func runReceive( webHandler.SetWriter(nil) webHandler.Hashring(h) msg := "hashring has changed; server is not ready to receive web requests." - statusProber.SetNotReady(errors.New(msg)) + statusProber.NotReady(errors.New(msg)) level.Info(logger).Log("msg", msg) updateDB <- struct{}{} case <-cancel: @@ -337,7 +337,16 @@ func runReceive( httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) - g.Add(srv.ListenAndServe, srv.Shutdown) + g.Add(func() error { + statusProber.Healthy() + + return srv.ListenAndServe() + }, func(err error) { + statusProber.NotReady(err) + defer statusProber.NotHealthy(err) + + srv.Shutdown(err) + }) level.Debug(logger).Log("msg", "setting up grpc server") { diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 9660dbd275..9ef8bd6107 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -500,7 +500,7 @@ func runRule( cancel() }) } - statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Start gRPC server. { store := store.NewTSDBStore(logger, reg, db, component.Rule, lset) @@ -515,11 +515,12 @@ func runRule( grpcserver.WithGracePeriod(grpcGracePeriod), grpcserver.WithTLSConfig(tlsCfg), ) + g.Add(func() error { - statusProber.SetReady() + statusProber.Ready() return s.ListenAndServe() }, func(err error) { - statusProber.SetNotReady(err) + statusProber.NotReady(err) s.Shutdown(err) }) } @@ -558,7 +559,16 @@ func runRule( ) srv.Handle("/", router) - g.Add(srv.ListenAndServe, srv.Shutdown) + g.Add(func() error { + statusProber.Healthy() + + return srv.ListenAndServe() + }, func(err error) { + statusProber.NotReady(err) + defer statusProber.NotHealthy(err) + + srv.Shutdown(err) + }) } confContentYaml, err := objStoreConfig.Content() diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 437e729f69..854e84c2bc 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -140,13 +140,22 @@ func runSidecar( } // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) srv := httpserver.New(logger, reg, comp, statusProber, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) - g.Add(srv.ListenAndServe, srv.Shutdown) + g.Add(func() error { + statusProber.Healthy() + + return srv.ListenAndServe() + }, func(err error) { + statusProber.NotReady(err) + defer statusProber.NotHealthy(err) + + srv.Shutdown(err) + }) // Setup all the concurrent groups. { @@ -179,7 +188,7 @@ func runSidecar( "err", err, ) promUp.Set(0) - statusProber.SetNotReady(err) + statusProber.NotReady(err) return err } @@ -188,7 +197,7 @@ func runSidecar( "external_labels", m.Labels().String(), ) promUp.Set(1) - statusProber.SetReady() + statusProber.Ready() lastHeartbeat.Set(float64(time.Now().UnixNano()) / 1e9) return nil }) @@ -247,10 +256,10 @@ func runSidecar( grpcserver.WithTLSConfig(tlsCfg), ) g.Add(func() error { - statusProber.SetReady() + statusProber.Ready() return s.ListenAndServe() }, func(err error) { - statusProber.SetNotReady(err) + statusProber.NotReady(err) s.Shutdown(err) }) } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 13833a3a12..51f4723d39 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -132,13 +132,22 @@ func runStore( advertiseCompatibilityLabel bool, ) error { // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + statusProber := prober.New(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) srv := httpserver.New(logger, reg, component, statusProber, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) - g.Add(srv.ListenAndServe, srv.Shutdown) + g.Add(func() error { + statusProber.Healthy() + + return srv.ListenAndServe() + }, func(err error) { + statusProber.NotReady(err) + defer statusProber.NotHealthy(err) + + srv.Shutdown(err) + }) confContentYaml, err := objStoreConfig.Content() if err != nil { @@ -241,10 +250,10 @@ func runStore( g.Add(func() error { <-bucketStoreReady - statusProber.SetReady() + statusProber.Ready() return s.ListenAndServe() }, func(err error) { - statusProber.SetNotReady(err) + statusProber.NotReady(err) s.Shutdown(err) }) } diff --git a/pkg/prober/prober.go b/pkg/prober/prober.go index 36bea35b88..c1d38fbd11 100644 --- a/pkg/prober/prober.go +++ b/pkg/prober/prober.go @@ -1,24 +1,22 @@ package prober import ( - "fmt" "io" "net/http" - "sync" + "sync/atomic" "github.com/prometheus/client_golang/prometheus" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "github.com/prometheus/common/route" "github.com/thanos-io/thanos/pkg/component" ) +type check func() bool + const ( - healthyEndpointPath = "/-/healthy" - readyEndpointPath = "/-/ready" - probeErrorHTTPStatus = 503 - initialErrorFmt = "thanos %s is initializing" + ready = "ready" + healthy = "healthy" ) // Prober represents health and readiness status of given component. @@ -36,135 +34,106 @@ const ( // and mitigate these situations. A pod with containers reporting that they are not ready // does not receive traffic through Kubernetes Services. type Prober struct { - logger log.Logger - component component.Component - readyMtx sync.RWMutex - readiness error - healthyMtx sync.RWMutex - healthiness error - readyStateMetric prometheus.Gauge - healthyStateMetric prometheus.Gauge + component component.Component + logger log.Logger + + ready uint32 + healthy uint32 + + status *prometheus.GaugeVec } -// NewProber returns Prober representing readiness and healthiness of given component. -func NewProber(component component.Component, logger log.Logger, reg prometheus.Registerer) *Prober { - initialErr := fmt.Errorf(initialErrorFmt, component) +// New returns Prober representing readiness and healthiness of given component. +func New(component component.Component, logger log.Logger, reg prometheus.Registerer) *Prober { p := &Prober{ - component: component, - logger: logger, - healthiness: initialErr, - readiness: initialErr, - readyStateMetric: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "prober_ready", - Help: "Represents readiness status of the component Prober.", + component: component, + logger: logger, + status: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "status", + Help: "Represents status (0 indicates success, 1 indicates failure) of the component.", ConstLabels: map[string]string{"component": component.String()}, - }), - healthyStateMetric: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "prober_healthy", - Help: "Represents health status of the component Prober.", - ConstLabels: map[string]string{"component": component.String()}, - }), + }, + []string{"check"}, + ), } + if reg != nil { - reg.MustRegister(p.readyStateMetric, p.healthyStateMetric) + reg.MustRegister(p.status) } + return p } -// RegisterInRouter registers readiness and liveness probes to router. -func (p *Prober) RegisterInRouter(router *route.Router) { - router.Get(healthyEndpointPath, p.probeHandlerFunc(p.IsHealthy, "healthy")) - router.Get(readyEndpointPath, p.probeHandlerFunc(p.IsReady, "ready")) +// HealthyHandler returns a HTTP Handler which responds health checks. +func (p *Prober) HealthyHandler() http.HandlerFunc { + return p.handler(p.isHealthy) } -// RegisterInMux registers readiness and liveness probes to mux. -func (p *Prober) RegisterInMux(mux *http.ServeMux) { - mux.HandleFunc(healthyEndpointPath, p.probeHandlerFunc(p.IsHealthy, "healthy")) - mux.HandleFunc(readyEndpointPath, p.probeHandlerFunc(p.IsReady, "ready")) +// ReadyHandler returns a HTTP Handler which responds readiness checks. +func (p *Prober) ReadyHandler() http.HandlerFunc { + return p.handler(p.isReady) } -func (p *Prober) writeResponse(w http.ResponseWriter, probeFn func() error, probeType string) { - if err := probeFn(); err != nil { - http.Error(w, fmt.Sprintf("thanos %v is not %v. Reason: %v", p.component, probeType, err), probeErrorHTTPStatus) - return - } - if _, err := io.WriteString(w, fmt.Sprintf("thanos %v is %v", p.component, probeType)); err != nil { - level.Error(p.logger).Log("msg", "failed to write probe response", "probe type", probeType, "err", err) +func (p *Prober) handler(c check) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + if !c() { + http.Error(w, "NOT OK", http.StatusServiceUnavailable) + return + } + if _, err := io.WriteString(w, "OK"); err != nil { + level.Error(p.logger).Log("msg", "failed to write probe response", "err", err) + } } } -func (p *Prober) probeHandlerFunc(probeFunc func() error, probeType string) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, _ *http.Request) { - p.writeResponse(w, probeFunc, probeType) - } +// isReady returns true if component is ready. +func (p *Prober) isReady() bool { + ready := atomic.LoadUint32(&p.ready) + return ready > 0 } -// IsReady returns error if component is not ready and nil otherwise. -func (p *Prober) IsReady() error { - p.readyMtx.RLock() - defer p.readyMtx.RUnlock() - return p.readiness +// isHealthy returns true if component is healthy. +func (p *Prober) isHealthy() bool { + healthy := atomic.LoadUint32(&p.healthy) + return healthy > 0 } -// SetReady sets components status to ready. -func (p *Prober) SetReady() { - p.readyMtx.Lock() - defer p.readyMtx.Unlock() - if p.readiness != nil { +// Ready sets components status to ready. +func (p *Prober) Ready() { + old := atomic.SwapUint32(&p.ready, 1) + + if old == 0 { + p.status.WithLabelValues(ready).Set(1) level.Info(p.logger).Log("msg", "changing probe status", "status", "ready") - p.readyStateMetric.Set(1) } - p.readiness = nil } -// SetNotReady sets components status to not ready with given error as a cause. -func (p *Prober) SetNotReady(err error) { - p.readyMtx.Lock() - defer p.readyMtx.Unlock() - if err != nil && p.readiness == nil { +// NotReady sets components status to not ready with given error as a cause. +func (p *Prober) NotReady(err error) { + old := atomic.SwapUint32(&p.ready, 0) + + if old == 1 { + p.status.WithLabelValues(ready).Set(0) level.Warn(p.logger).Log("msg", "changing probe status", "status", "not-ready", "reason", err) - p.readyStateMetric.Set(0) } - p.readiness = err } -// IsHealthy returns error if component is not healthy and nil if it is. -func (p *Prober) IsHealthy() error { - p.healthyMtx.RLock() - defer p.healthyMtx.RUnlock() - return p.healthiness -} +// Healthy sets components status to healthy. +func (p *Prober) Healthy() { + old := atomic.SwapUint32(&p.healthy, 1) -// SetHealthy sets components status to healthy. -func (p *Prober) SetHealthy() { - p.healthyMtx.Lock() - defer p.healthyMtx.Unlock() - if p.healthiness != nil { + if old == 0 { + p.status.WithLabelValues(healthy).Set(1) level.Info(p.logger).Log("msg", "changing probe status", "status", "healthy") - p.healthyStateMetric.Set(1) } - p.healthiness = nil } -// SetNotHealthy sets components status to not healthy with given error as a cause. -func (p *Prober) SetNotHealthy(err error) { - p.healthyMtx.Lock() - defer p.healthyMtx.Unlock() - if err != nil && p.healthiness == nil { - level.Warn(p.logger).Log("msg", "changing probe status", "status", "unhealthy", "reason", err) - p.healthyStateMetric.Set(0) - } - p.healthiness = err -} +// NotHealthy sets components status to not healthy with given error as a cause. +func (p *Prober) NotHealthy(err error) { + old := atomic.SwapUint32(&p.healthy, 0) -// HandleIfReady if probe is ready calls the function otherwise returns 503. -func (p *Prober) HandleIfReady(f http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - ready := p.IsReady() - if ready == nil { - f(w, r) - return - } - p.writeResponse(w, func() error { return ready }, "ready") + if old == 1 { + p.status.WithLabelValues(healthy).Set(0) + level.Info(p.logger).Log("msg", "changing probe status", "status", "not-healthy", "reason", err) } } diff --git a/pkg/prober/prober_test.go b/pkg/prober/prober_test.go index b7e31f412c..694bc105d2 100644 --- a/pkg/prober/prober_test.go +++ b/pkg/prober/prober_test.go @@ -7,19 +7,18 @@ import ( "net/http" "path" "testing" - "time" "github.com/go-kit/kit/log" "github.com/oklog/run" - "github.com/pkg/errors" - "github.com/prometheus/common/route" - "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" ) -func queryHTTPGetEndpoint(ctx context.Context, t *testing.T, logger log.Logger, url string) (*http.Response, error) { +func doGet(ctx context.Context, url string) (*http.Response, error) { req, err := http.NewRequest("GET", fmt.Sprintf("http://%s", url), nil) - testutil.Ok(t, err) + if err != nil { + return nil, err + } + return http.DefaultClient.Do(req.WithContext(ctx)) } @@ -32,141 +31,109 @@ func (c TestComponent) String() string { } func TestProberHealthInitialState(t *testing.T) { - component := TestComponent{name: "test"} - expectedErrorMessage := fmt.Sprintf(initialErrorFmt, component) - p := NewProber(component, log.NewNopLogger(), nil) + p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) - err := p.IsHealthy() - testutil.NotOk(t, err) - testutil.Equals(t, err.Error(), expectedErrorMessage) + testutil.Assert(t, !p.isHealthy(), "initially should not be healthy") } func TestProberReadinessInitialState(t *testing.T) { - component := TestComponent{name: "test"} - expectedErrorMessage := fmt.Sprintf(initialErrorFmt, component) - p := NewProber(component, log.NewNopLogger(), nil) + p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) - err := p.IsReady() - testutil.NotOk(t, err) - testutil.Equals(t, err.Error(), expectedErrorMessage) + testutil.Assert(t, !p.isReady(), "initially should not be ready") } -func TestProberReadyStatusSetting(t *testing.T) { - component := TestComponent{name: "test"} +func TestProberHealthyStatusSetting(t *testing.T) { testError := fmt.Errorf("test error") - p := NewProber(component, log.NewNopLogger(), nil) - - p.SetReady() - err := p.IsReady() - testutil.Equals(t, err, nil) - p.SetNotReady(testError) - err = p.IsReady() - testutil.NotOk(t, err) + p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) + + p.Healthy() + + testutil.Assert(t, p.isHealthy(), "should be healthy") + + p.NotHealthy(testError) + + testutil.Assert(t, !p.isHealthy(), "should not be healthy") } -func TestProberHeatlthyStatusSetting(t *testing.T) { - component := TestComponent{name: "test"} +func TestProberReadyStatusSetting(t *testing.T) { testError := fmt.Errorf("test error") - p := NewProber(component, log.NewNopLogger(), nil) - - p.SetHealthy() - err := p.IsHealthy() - testutil.Equals(t, err, nil) - p.SetNotHealthy(testError) - err = p.IsHealthy() - testutil.NotOk(t, err) + p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) + + p.Ready() + + testutil.Assert(t, p.isReady(), "should be ready") + + p.NotReady(testError) + + testutil.Assert(t, !p.isReady(), "should not be ready") } func TestProberMuxRegistering(t *testing.T) { - component := TestComponent{name: "test"} - ctx := context.Background() - var g run.Group - mux := http.NewServeMux() - - freePort, err := testutil.FreePort() - testutil.Ok(t, err) - serverAddress := fmt.Sprintf("localhost:%d", freePort) + serverAddress := fmt.Sprintf("localhost:%d", 8081) l, err := net.Listen("tcp", serverAddress) testutil.Ok(t, err) - g.Add(func() error { - return errors.Wrap(http.Serve(l, mux), "serve probes") - }, func(error) {}) - p := NewProber(component, log.NewNopLogger(), nil) - p.RegisterInMux(mux) + p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) - go func() { _ = g.Run() }() + healthyEndpointPath := "/-/healthy" + readyEndpointPath := "/-/ready" - testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { - resp, err := queryHTTPGetEndpoint(ctx, t, log.NewNopLogger(), path.Join(serverAddress, healthyEndpointPath)) - testutil.Ok(t, err) - testutil.Equals(t, probeErrorHTTPStatus, resp.StatusCode) - return err - })) - - testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { - resp, err := queryHTTPGetEndpoint(ctx, t, log.NewNopLogger(), path.Join(serverAddress, readyEndpointPath)) - testutil.Ok(t, err) - testutil.Equals(t, probeErrorHTTPStatus, resp.StatusCode) - return err - })) + mux := http.NewServeMux() + mux.HandleFunc(healthyEndpointPath, p.HealthyHandler()) + mux.HandleFunc(readyEndpointPath, p.ReadyHandler()) - p.SetHealthy() + var g run.Group + g.Add(func() error { + return fmt.Errorf("serve probes %w", http.Serve(l, mux)) + }, func(err error) { + t.Fatalf("server failed: %v", err) + }) - testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { - resp, err := queryHTTPGetEndpoint(ctx, t, log.NewNopLogger(), path.Join(serverAddress, healthyEndpointPath)) - testutil.Ok(t, err) - testutil.Equals(t, 200, resp.StatusCode) - return err - })) + go func() { _ = g.Run() }() -} + { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() -func TestProberRouterRegistering(t *testing.T) { - component := TestComponent{name: "test"} - router := route.New() - ctx := context.Background() - var g run.Group - mux := http.NewServeMux() + resp, err := doGet(ctx, path.Join(serverAddress, healthyEndpointPath)) + testutil.Ok(t, err) + defer resp.Body.Close() - freePort, err := testutil.FreePort() - testutil.Ok(t, err) - serverAddress := fmt.Sprintf("localhost:%d", freePort) + testutil.Equals(t, resp.StatusCode, http.StatusServiceUnavailable, "should not be healthy, response code: %d", resp.StatusCode) + } + { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - l, err := net.Listen("tcp", serverAddress) - testutil.Ok(t, err) - g.Add(func() error { - return errors.Wrap(http.Serve(l, mux), "serve probes") - }, func(error) {}) + resp, err := doGet(ctx, path.Join(serverAddress, readyEndpointPath)) + testutil.Ok(t, err) + defer resp.Body.Close() - p := NewProber(component, log.NewNopLogger(), nil) - p.RegisterInRouter(router) - mux.Handle("/", router) + testutil.Equals(t, resp.StatusCode, http.StatusServiceUnavailable, "should not be ready, response code: %d", resp.StatusCode) + } + { + p.Healthy() - go func() { _ = g.Run() }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { - resp, err := queryHTTPGetEndpoint(ctx, t, log.NewNopLogger(), path.Join(serverAddress, healthyEndpointPath)) + resp, err := doGet(ctx, path.Join(serverAddress, healthyEndpointPath)) testutil.Ok(t, err) - testutil.Equals(t, probeErrorHTTPStatus, resp.StatusCode) - return err - })) + defer resp.Body.Close() - testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { - resp, err := queryHTTPGetEndpoint(ctx, t, log.NewNopLogger(), path.Join(serverAddress, readyEndpointPath)) - testutil.Ok(t, err) - testutil.Equals(t, probeErrorHTTPStatus, resp.StatusCode) - return err - })) + testutil.Equals(t, resp.StatusCode, http.StatusOK, "should be healthy, response code: %d", resp.StatusCode) + } + { + p.Ready() - p.SetHealthy() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { - resp, err := queryHTTPGetEndpoint(ctx, t, log.NewNopLogger(), path.Join(serverAddress, healthyEndpointPath)) + resp, err := doGet(ctx, path.Join(serverAddress, readyEndpointPath)) testutil.Ok(t, err) - testutil.Equals(t, 200, resp.StatusCode) - return err - })) + defer resp.Body.Close() + testutil.Equals(t, resp.StatusCode, http.StatusOK, "should be ready, response code: %d", resp.StatusCode) + } } diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index 290d97c59a..56e7e87268 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -35,8 +35,8 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, mux := http.NewServeMux() registerMetrics(mux, reg) + registerProbes(mux, prober) registerProfiler(mux) - prober.RegisterInMux(mux) return &Server{ logger: log.With(logger, "service", "http/server", "component", comp.String()), @@ -50,7 +50,6 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, // ListenAndServe listens on the TCP network address and handles requests on incoming connections. func (s *Server) ListenAndServe() error { - s.prober.SetHealthy() level.Info(s.logger).Log("msg", "listening for requests and metrics", "address", s.opts.listen) return errors.Wrap(s.srv.ListenAndServe(), "serve HTTP and metrics") } @@ -58,9 +57,6 @@ func (s *Server) ListenAndServe() error { // Shutdown gracefully shuts down the server by waiting, // for specified amount of time (by gracePeriod) for connections to return to idle and then shut down. func (s *Server) Shutdown(err error) { - s.prober.SetNotReady(err) - defer s.prober.SetNotHealthy(err) - if err == http.ErrServerClosed { level.Warn(s.logger).Log("msg", "internal server closed unexpectedly") return @@ -95,5 +91,14 @@ func registerProfiler(mux *http.ServeMux) { } func registerMetrics(mux *http.ServeMux, g prometheus.Gatherer) { - mux.Handle("/metrics", promhttp.HandlerFor(g, promhttp.HandlerOpts{})) + if g != nil { + mux.Handle("/metrics", promhttp.HandlerFor(g, promhttp.HandlerOpts{})) + } +} + +func registerProbes(mux *http.ServeMux, p *prober.Prober) { + if p != nil { + mux.Handle("/-/healthy", p.HealthyHandler()) + mux.Handle("/-/ready", p.ReadyHandler()) + } } From 96c3657784bc9be1b9ae77718e6f6ccc461c9cd0 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Tue, 26 Nov 2019 15:43:15 +0100 Subject: [PATCH 065/257] pkg/rule: support identical rule filenames in different directories (#1791) Signed-off-by: Simon Pasquier --- CHANGELOG.md | 1 + pkg/rule/rule.go | 3 ++- pkg/rule/rule_test.go | 39 +++++++++++++++++++++++++++------------ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0312950027..a8b2880b17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1773](https://github.com/thanos-io/thanos/pull/1773) Thanos Ruler: fixed the /api/v1/rules endpoint that returned 500 status code with `failed to assert type of rule ...` message. - [#1770](https://github.com/thanos-io/thanos/pull/1770) Fix `--web.external-prefix` 404s for static resources. - [#1785](https://github.com/thanos-io/thanos/pull/1785) Thanos Ruler: the /api/v1/rules endpoints now returns the original rule filenames. +- [#1791](https://github.com/thanos-io/thanos/pull/1791) Thanos Ruler now supports identical rule filenames in different directories. ### Changed diff --git a/pkg/rule/rule.go b/pkg/rule/rule.go index 970ffbe8bc..a473b517f4 100644 --- a/pkg/rule/rule.go +++ b/pkg/rule/rule.go @@ -1,6 +1,7 @@ package thanosrule import ( + "crypto/sha256" "fmt" "io/ioutil" "os" @@ -187,7 +188,7 @@ func (m *Manager) Update(evalInterval time.Duration, files []string) error { continue } - newFn := filepath.Join(m.workDir, filepath.Base(fn)+"."+s.String()) + newFn := filepath.Join(m.workDir, fmt.Sprintf("%s.%x.%s", filepath.Base(fn), sha256.Sum256([]byte(fn)), s.String())) if err := ioutil.WriteFile(newFn, b, os.ModePerm); err != nil { errs = append(errs, errors.Wrap(err, newFn)) continue diff --git a/pkg/rule/rule_test.go b/pkg/rule/rule_test.go index 8b9864a17e..9c293976a2 100644 --- a/pkg/rule/rule_test.go +++ b/pkg/rule/rule_test.go @@ -3,7 +3,6 @@ package thanosrule import ( "io/ioutil" "os" - "path" "path/filepath" "sort" "strings" @@ -22,15 +21,17 @@ func TestUpdate(t *testing.T) { dir, err := ioutil.TempDir("", "test_rule_rule_groups") testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() + err = os.MkdirAll(filepath.Join(dir, "subdir"), 0775) + testutil.Ok(t, err) - testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "no_strategy.yaml"), []byte(` + testutil.Ok(t, ioutil.WriteFile(filepath.Join(dir, "no_strategy.yaml"), []byte(` groups: - name: "something1" rules: - alert: "some" expr: "up" `), os.ModePerm)) - testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "abort.yaml"), []byte(` + testutil.Ok(t, ioutil.WriteFile(filepath.Join(dir, "abort.yaml"), []byte(` groups: - name: "something2" partial_response_strategy: "abort" @@ -38,7 +39,7 @@ groups: - alert: "some" expr: "up" `), os.ModePerm)) - testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "warn.yaml"), []byte(` + testutil.Ok(t, ioutil.WriteFile(filepath.Join(dir, "warn.yaml"), []byte(` groups: - name: "something3" partial_response_strategy: "warn" @@ -46,7 +47,7 @@ groups: - alert: "some" expr: "up" `), os.ModePerm)) - testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "wrong.yaml"), []byte(` + testutil.Ok(t, ioutil.WriteFile(filepath.Join(dir, "wrong.yaml"), []byte(` groups: - name: "something4" partial_response_strategy: "afafsdgsdgs" # Err 1 @@ -54,7 +55,7 @@ groups: - alert: "some" expr: "up" `), os.ModePerm)) - testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "combined.yaml"), []byte(` + testutil.Ok(t, ioutil.WriteFile(filepath.Join(dir, "combined.yaml"), []byte(` groups: - name: "something5" partial_response_strategy: "warn" @@ -71,6 +72,14 @@ groups: - alert: "some" expr: "up" `), os.ModePerm)) + // Same filename as the first rule file but different path. + testutil.Ok(t, ioutil.WriteFile(filepath.Join(dir, "subdir", "no_strategy.yaml"), []byte(` +groups: +- name: "something8" + rules: + - alert: "some" + expr: "up" +`), os.ModePerm)) opts := rules.ManagerOptions{ Logger: log.NewLogfmtLogger(os.Stderr), @@ -80,12 +89,13 @@ groups: m.SetRuleManager(storepb.PartialResponseStrategy_WARN, rules.NewManager(&opts)) err = m.Update(10*time.Second, []string{ - path.Join(dir, "no_strategy.yaml"), - path.Join(dir, "abort.yaml"), - path.Join(dir, "warn.yaml"), - path.Join(dir, "wrong.yaml"), - path.Join(dir, "combined.yaml"), - path.Join(dir, "non_existing.yaml"), + filepath.Join(dir, "no_strategy.yaml"), + filepath.Join(dir, "abort.yaml"), + filepath.Join(dir, "warn.yaml"), + filepath.Join(dir, "wrong.yaml"), + filepath.Join(dir, "combined.yaml"), + filepath.Join(dir, "non_existing.yaml"), + filepath.Join(dir, "subdir", "no_strategy.yaml"), }) testutil.NotOk(t, err) @@ -132,6 +142,11 @@ groups: file: filepath.Join(dir, "combined.yaml"), strategy: storepb.PartialResponseStrategy_ABORT, }, + { + name: "something8", + file: filepath.Join(dir, "subdir", "no_strategy.yaml"), + strategy: storepb.PartialResponseStrategy_ABORT, + }, } testutil.Equals(t, len(exp), len(g)) for i := range exp { From 7e11afe64af0c096743a3de8a594616abf52be45 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 26 Nov 2019 08:38:13 -0800 Subject: [PATCH 066/257] Update prometheus for subqueries support in ruler. (#1790) * Updated Prometheus dep to 2.14.0 tip (80ba03c67da11403fefa2d48c4dd30af0fee9fc0) for subqueries support in ruler. Signed-off-by: Bartek Plotka * Added rule test against regression. Signed-off-by: Bartek Plotka --- CHANGELOG.md | 1 + cmd/thanos/bucket.go | 2 +- cmd/thanos/main_test.go | 2 +- cmd/thanos/query.go | 2 +- cmd/thanos/receive.go | 2 +- cmd/thanos/rule.go | 8 +-- cmd/thanos/sidecar.go | 2 +- go.mod | 7 +- go.sum | 60 +++++++++++++--- pkg/block/block_test.go | 2 +- pkg/block/index.go | 2 +- pkg/block/index_test.go | 2 +- pkg/compact/compact.go | 4 +- pkg/compact/compact_e2e_test.go | 6 +- pkg/compact/downsample/downsample.go | 2 +- pkg/compact/downsample/downsample_test.go | 2 +- .../downsample/streamed_block_writer.go | 2 +- pkg/promclient/promclient.go | 2 +- pkg/promclient/promclient_e2e_test.go | 2 +- pkg/query/api/v1_test.go | 53 +++++++------- .../test-storeset-pre-v0.8.0/storeset.go | 2 +- pkg/query/storeset.go | 2 +- pkg/query/storeset_test.go | 14 ++-- pkg/receive/tsdb_test.go | 4 +- pkg/rule/rule_test.go | 71 ++++++++++++++++++- pkg/shipper/shipper.go | 4 +- pkg/shipper/shipper_e2e_test.go | 2 +- pkg/store/bucket.go | 31 ++++---- pkg/store/bucket_e2e_test.go | 2 +- pkg/store/bucket_test.go | 50 ++++++------- pkg/store/cache/cache.go | 2 +- pkg/store/cache/cache_test.go | 2 +- pkg/store/matchers.go | 18 ++--- pkg/store/prometheus.go | 2 +- pkg/store/prometheus_test.go | 2 +- pkg/store/proxy.go | 2 +- pkg/store/proxy_test.go | 11 ++- pkg/store/tsdb.go | 2 +- pkg/store/tsdb_test.go | 2 +- pkg/testutil/prometheus.go | 2 +- test/e2e/store_gateway_test.go | 2 +- 41 files changed, 246 insertions(+), 148 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8b2880b17..4b27baed89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel is now exposed by new metric: `thanos_compact_group_compaction_runs_started_total` and `thanos_compact_group_compaction_runs_completed_total` which counts compaction runs overall. - [#1748](https://github.com/thanos-io/thanos/pull/1748) Updated all dependencies. - [#1694](https://github.com/thanos-io/thanos/pull/1694) `prober_ready` and `prober_healthy` metrics are removed, for sake of `status`. Now `status` exposes same metric with a label, `check`. `check` can have "healty" or "ready" depending on status of the probe. +- [#1790](https://github.com/thanos-io/thanos/pull/1790) Fixes subqueries support for ruler. ## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14 diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 4d1e4e92ca..157706ef3a 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -19,7 +19,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/route" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact" diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index a526f0cfb1..ea4f7063e1 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -16,7 +16,7 @@ import ( "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/compact/downsample" diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index f75c23f45b..2dd9c17663 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -19,8 +19,8 @@ import ( "github.com/prometheus/common/route" "github.com/prometheus/prometheus/discovery/file" "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/discovery/cache" "github.com/thanos-io/thanos/pkg/discovery/dns" diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 5571dbd88b..09c3e8f317 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -14,8 +14,8 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage/tsdb" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 9ef8bd6107..b3f3db72ef 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -27,11 +27,10 @@ import ( "github.com/prometheus/common/route" "github.com/prometheus/prometheus/discovery/file" "github.com/prometheus/prometheus/discovery/targetgroup" - promlabels "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/storage/tsdb" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/prometheus/prometheus/util/strutil" "github.com/thanos-io/thanos/pkg/alert" "github.com/thanos-io/thanos/pkg/block/metadata" @@ -327,6 +326,7 @@ func runRule( ResendDelay: resendDelay, } + // TODO(bwplotka): Hide this behind thanos rules.Manager. for _, strategy := range storepb.PartialResponseStrategy_value { s := storepb.PartialResponseStrategy(strategy) @@ -718,9 +718,9 @@ func parseFlagLabels(s []string) (labels.Labels, error) { return lset, nil } -func labelsTSDBToProm(lset labels.Labels) (res promlabels.Labels) { +func labelsTSDBToProm(lset labels.Labels) (res labels.Labels) { for _, l := range lset { - res = append(res, promlabels.Label{ + res = append(res, labels.Label{ Name: l.Name, Value: l.Value, }) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 854e84c2bc..f4b319898c 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -14,7 +14,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" diff --git a/go.mod b/go.mod index 55a8b9c175..3a91fe1b1b 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 github.com/prometheus/common v0.7.0 github.com/prometheus/procfs v0.0.6 // indirect - github.com/prometheus/prometheus v1.8.2-0.20191114185310-85cae37107c2 // Prometheus master v2.14.0 + github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1 // Prometheus master v2.14.0 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/smartystreets/assertions v1.0.1 // indirect @@ -89,10 +89,9 @@ require ( golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 // indirect golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect - golang.org/x/tools v0.0.0-20191114222411-4191b8cbba09 // indirect - google.golang.org/api v0.13.0 + google.golang.org/api v0.14.0 google.golang.org/appengine v1.6.5 // indirect - google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de + google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9 google.golang.org/grpc v1.25.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index dbbe98e074..b8c2309441 100644 --- a/go.sum +++ b/go.sum @@ -5,13 +5,14 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.48.0 h1:6ZHYIRlohUdU4LrLHbTsReY1eYy/MoZW1FsEyBuMXsk= -cloud.google.com/go v0.48.0/go.mod h1:gGOnoa/XMQYHAscREBlbdHduGchEaP9N0//OXdrPI/M= cloud.google.com/go v0.49.0 h1:CH+lkubJzcPYB1Ggupcq0+k8Ni2ILdG2lYjDIgavDBQ= cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.3.0 h1:2Ze/3nQD5F+HfL0xOPM2EeawDWs+NPRtzgcre+17iZU= @@ -39,6 +40,7 @@ github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSW github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/to v0.3.1-0.20191028180845-3492b2aff503 h1:2McfZNaDqGPjv2pddK547PENIk4HV+NT7gvqRq4L0us= github.com/Azure/go-autorest/autorest/to v0.3.1-0.20191028180845-3492b2aff503/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= @@ -55,6 +57,7 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.6 h1:U68crOE3y3MPttCMQGywZOLrTeF5HHJ3/vDBCJn9/bA= github.com/OneOfOne/xxhash v1.2.6/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -83,6 +86,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/aws/aws-sdk-go v1.23.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.35 h1:fe2tJnqty/v/50pyngKdNk/NP8PFphYDA1Z7N3EiiiE= github.com/aws/aws-sdk-go v1.25.35/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -100,6 +104,7 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -121,14 +126,18 @@ github.com/elastic/go-sysinfo v1.0.1/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyz github.com/elastic/go-sysinfo v1.1.1 h1:ZVlaLDyhVkDfjwPGU55CQRCRolNpc7P0BbyhhQZQmMI= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/structtag v1.1.0 h1:6j4mUV/ES2duvnAzKMFkN6/A5mCaNYPD3xfbAkLLOF8= github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -164,6 +173,7 @@ github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -172,6 +182,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -189,6 +200,7 @@ github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -199,6 +211,7 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -220,6 +233,7 @@ github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEo github.com/gophercloud/gophercloud v0.6.0 h1:Xb2lcqZtml1XjgYZxbeayEemq7ASbeTp09m36gQFpEU= github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de h1:F7WD09S8QB4LrkEpka0dFPLSotH11HRpCsLIbIcJ7sU= github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -236,7 +250,9 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/api v1.3.0 h1:HXNYlRkkM/t+Y/Yhxtwcy02dlYwIaoxzvxPnS+cqy78= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0 h1:UOxjlb4xVNF93jak1mzzoBatyFju9nrkxpVwIp/QqxQ= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= @@ -245,16 +261,20 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.1 h1:DMo4fmknnz0E0evoNYnV48RjWndOsmd6OW+09R3cEP8= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -264,10 +284,12 @@ github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM= github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc= github.com/hashicorp/serf v0.8.5/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= @@ -287,6 +309,7 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= @@ -295,16 +318,18 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/leanovate/gopter v0.2.4 h1:U4YLBggDFhJdqQsG4Na2zX7joVTky9vHaj/AGEwSuXU= github.com/leanovate/gopter v0.2.4/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743 h1:143Bb8f8DuGWck/xpNUOckBVYfFbBTnLevfRZ1aVVqo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.17.1 h1:PgitbgUDool2AcHopDNTlvwq7BQeZssTGs4EVwcGhr8= -github.com/lightstep/lightstep-tracer-go v0.17.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lightstep/lightstep-tracer-go v0.18.0 h1:fAazJekOWnfBeQYwk9jEgIWWKmBxq4ev3WfsAnezgc4= github.com/lightstep/lightstep-tracer-go v0.18.0/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lovoo/gcloud-opentracing v0.3.0 h1:nAeKG70rIsog0TelcEtt6KU0Y1s5qXtsDLnHp0urPLU= @@ -333,6 +358,7 @@ github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceT github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -364,9 +390,11 @@ github.com/olekukonko/tablewriter v0.0.2 h1:sq53g+DWf0J6/ceFUHpQ0nAEb6WgM++fq16M github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= @@ -376,6 +404,7 @@ github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsq github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -383,6 +412,7 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/alertmanager v0.18.0/go.mod h1:WcxHBl40VSPuOaqWae6l6HpnEOVRIycEJ7i9iYkadEE= @@ -413,8 +443,8 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.6 h1:0qbH+Yqu/cj1ViVLvEWCP6qMQ4efWUj6bQqOEA0V0U4= github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= -github.com/prometheus/prometheus v1.8.2-0.20191114185310-85cae37107c2 h1:33wX5JpFUQiXXSWO/g8a9MwYj8VzEno7s5bpJpVEtN8= -github.com/prometheus/prometheus v1.8.2-0.20191114185310-85cae37107c2/go.mod h1:PVTKYlgELGIDbIKIyWRzD4WKjnavPynGOFLSuDpvOwU= +github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1 h1:5ee1ewJCJYB7Bp314qaPcRNFaAPsdHN6BFzBC1wMVbQ= +github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1/go.mod h1:PVTKYlgELGIDbIKIyWRzD4WKjnavPynGOFLSuDpvOwU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -425,9 +455,12 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo github.com/samuel/go-zookeeper v0.0.0-20190810000440-0ceca61e4d75/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= @@ -436,20 +469,25 @@ github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= @@ -586,9 +624,7 @@ golang.org/x/tools v0.0.0-20190918214516-5a1a30219888/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191111182352-50fa39b762bc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191114222411-4191b8cbba09/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -599,11 +635,13 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0 h1:uMf5uLi4eQMRrMKhCplNik4U4H8Z6C1br3zOtAa/aDE= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -617,8 +655,7 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de h1:dFEMUWudT9iV1JMk6i6NwbfIw2V/2VDFyDYCZFypRxE= -google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9 h1:6XzpBoANz1NqMNfDXzc2QmHmbb1vyMsvRfoP5rM+K1I= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -635,6 +672,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -648,6 +686,7 @@ gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -673,6 +712,7 @@ k8s.io/client-go v0.0.0-20190620085101-78d2af792bab h1:E8Fecph0qbNsAbijJJQryKu4O k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k= k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6 h1:p0Ai3qVtkbCG/Af26dBmU0E1W58NID3hSSh7cMyylpM= diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index 8e14a841e4..51f07d13fe 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -13,7 +13,7 @@ import ( "github.com/fortytw2/leaktest" "github.com/go-kit/kit/log" "github.com/pkg/errors" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/testutil" diff --git a/pkg/block/index.go b/pkg/block/index.go index b2fd6791c7..4789bea794 100644 --- a/pkg/block/index.go +++ b/pkg/block/index.go @@ -20,10 +20,10 @@ import ( "github.com/go-kit/kit/log/level" "github.com/oklog/ulid" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/index" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/runutil" ) diff --git a/pkg/block/index_test.go b/pkg/block/index_test.go index 0aa4d70752..b562ffbe69 100644 --- a/pkg/block/index_test.go +++ b/pkg/block/index_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/go-kit/kit/log" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/testutil" ) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index f5d932b386..37ca4fdf76 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -16,11 +16,11 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/labels" promlables "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb" terrors "github.com/prometheus/prometheus/tsdb/errors" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact/downsample" @@ -564,7 +564,7 @@ func (cg *Group) Add(meta *metadata.Meta) error { cg.mtx.Lock() defer cg.mtx.Unlock() - if !cg.labels.Equals(labels.FromMap(meta.Thanos.Labels)) { + if !labels.Equal(cg.labels, labels.FromMap(meta.Thanos.Labels)) { return errors.New("block and group labels do not match") } if cg.resolution != meta.Thanos.Downsample.Resolution { diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 7513a944e4..f0a4b9e139 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -19,10 +19,10 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/index" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/objstore" @@ -387,7 +387,7 @@ func TestGroup_Compact_e2e(t *testing.T) { testutil.Equals(t, []ulid.ULID{metas[0].ULID, metas[1].ULID, metas[2].ULID}, meta.Compaction.Sources) // Check thanos meta. - testutil.Assert(t, extLabels.Equals(labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") + testutil.Assert(t, labels.Equal(extLabels, labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") testutil.Equals(t, int64(124), meta.Thanos.Downsample.Resolution) } { @@ -402,7 +402,7 @@ func TestGroup_Compact_e2e(t *testing.T) { testutil.Equals(t, []ulid.ULID{metas[6].ULID, metas[7].ULID}, meta.Compaction.Sources) // Check thanos meta. - testutil.Assert(t, extLabels2.Equals(labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") + testutil.Assert(t, labels.Equal(extLabels2, labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") testutil.Equals(t, int64(124), meta.Thanos.Downsample.Resolution) } }) diff --git a/pkg/compact/downsample/downsample.go b/pkg/compact/downsample/downsample.go index 4f223064c6..f51ce7cb02 100644 --- a/pkg/compact/downsample/downsample.go +++ b/pkg/compact/downsample/downsample.go @@ -10,13 +10,13 @@ import ( "github.com/go-kit/kit/log" "github.com/oklog/ulid" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/value" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" tsdberrors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/index" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/runutil" ) diff --git a/pkg/compact/downsample/downsample_test.go b/pkg/compact/downsample/downsample_test.go index 34e4eaa26f..3e0de64351 100644 --- a/pkg/compact/downsample/downsample_test.go +++ b/pkg/compact/downsample/downsample_test.go @@ -12,12 +12,12 @@ import ( "github.com/fortytw2/leaktest" "github.com/go-kit/kit/log" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/value" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/index" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/prometheus/prometheus/tsdb/tombstones" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" diff --git a/pkg/compact/downsample/streamed_block_writer.go b/pkg/compact/downsample/streamed_block_writer.go index d1f3f15f3f..0bc84e11c4 100644 --- a/pkg/compact/downsample/streamed_block_writer.go +++ b/pkg/compact/downsample/streamed_block_writer.go @@ -7,12 +7,12 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunks" tsdberrors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/runutil" diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 7c787b79d0..582c8a9721 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -25,10 +25,10 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" promlabels "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/tracing" diff --git a/pkg/promclient/promclient_e2e_test.go b/pkg/promclient/promclient_e2e_test.go index a1b742c9bd..bee767e963 100644 --- a/pkg/promclient/promclient_e2e_test.go +++ b/pkg/promclient/promclient_e2e_test.go @@ -13,8 +13,8 @@ import ( "github.com/go-kit/kit/log" "github.com/oklog/ulid" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" ) diff --git a/pkg/query/api/v1_test.go b/pkg/query/api/v1_test.go index dbeb56f05d..c4be888a24 100644 --- a/pkg/query/api/v1_test.go +++ b/pkg/query/api/v1_test.go @@ -36,7 +36,6 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/promql" - tsdb_labels "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/component" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" @@ -48,38 +47,38 @@ import ( func TestEndpoints(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() - lbls := []tsdb_labels.Labels{ - tsdb_labels.Labels{ - tsdb_labels.Label{Name: "__name__", Value: "test_metric1"}, - tsdb_labels.Label{Name: "foo", Value: "bar"}, + lbls := []labels.Labels{ + labels.Labels{ + labels.Label{Name: "__name__", Value: "test_metric1"}, + labels.Label{Name: "foo", Value: "bar"}, }, - tsdb_labels.Labels{ - tsdb_labels.Label{Name: "__name__", Value: "test_metric1"}, - tsdb_labels.Label{Name: "foo", Value: "boo"}, + labels.Labels{ + labels.Label{Name: "__name__", Value: "test_metric1"}, + labels.Label{Name: "foo", Value: "boo"}, }, - tsdb_labels.Labels{ - tsdb_labels.Label{Name: "__name__", Value: "test_metric2"}, - tsdb_labels.Label{Name: "foo", Value: "boo"}, + labels.Labels{ + labels.Label{Name: "__name__", Value: "test_metric2"}, + labels.Label{Name: "foo", Value: "boo"}, }, - tsdb_labels.Labels{ - tsdb_labels.Label{Name: "__name__", Value: "test_metric_replica1"}, - tsdb_labels.Label{Name: "foo", Value: "bar"}, - tsdb_labels.Label{Name: "replica", Value: "a"}, + labels.Labels{ + labels.Label{Name: "__name__", Value: "test_metric_replica1"}, + labels.Label{Name: "foo", Value: "bar"}, + labels.Label{Name: "replica", Value: "a"}, }, - tsdb_labels.Labels{ - tsdb_labels.Label{Name: "__name__", Value: "test_metric_replica1"}, - tsdb_labels.Label{Name: "foo", Value: "boo"}, - tsdb_labels.Label{Name: "replica", Value: "a"}, + labels.Labels{ + labels.Label{Name: "__name__", Value: "test_metric_replica1"}, + labels.Label{Name: "foo", Value: "boo"}, + labels.Label{Name: "replica", Value: "a"}, }, - tsdb_labels.Labels{ - tsdb_labels.Label{Name: "__name__", Value: "test_metric_replica1"}, - tsdb_labels.Label{Name: "foo", Value: "boo"}, - tsdb_labels.Label{Name: "replica", Value: "b"}, + labels.Labels{ + labels.Label{Name: "__name__", Value: "test_metric_replica1"}, + labels.Label{Name: "foo", Value: "boo"}, + labels.Label{Name: "replica", Value: "b"}, }, - tsdb_labels.Labels{ - tsdb_labels.Label{Name: "__name__", Value: "test_metric_replica1"}, - tsdb_labels.Label{Name: "foo", Value: "boo"}, - tsdb_labels.Label{Name: "replica1", Value: "a"}, + labels.Labels{ + labels.Label{Name: "__name__", Value: "test_metric_replica1"}, + labels.Label{Name: "foo", Value: "boo"}, + labels.Label{Name: "replica1", Value: "a"}, }, } diff --git a/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go b/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go index 041796d9f5..254491b690 100644 --- a/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go +++ b/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go @@ -16,7 +16,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store" diff --git a/pkg/query/storeset.go b/pkg/query/storeset.go index ebfa441611..6239e5d47e 100644 --- a/pkg/query/storeset.go +++ b/pkg/query/storeset.go @@ -12,7 +12,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store" diff --git a/pkg/query/storeset_test.go b/pkg/query/storeset_test.go index 54f9d0886f..6deba8e423 100644 --- a/pkg/query/storeset_test.go +++ b/pkg/query/storeset_test.go @@ -453,22 +453,22 @@ func TestStoreSet_Update(t *testing.T) { // Check stats. expected = newStoreAPIStats() expected[component.StoreAPI(nil)] = map[string]int{ - "{l1=\"no-store-type\",l2=\"v3\"}": 1, + "{l1=\"no-store-type\", l2=\"v3\"}": 1, } expected[component.Query] = map[string]int{ - "{l1=\"v2\",l2=\"v3\"}": 1, - "{l1=\"v2\",l2=\"v3\"},{l3=\"v4\"}": 2, + "{l1=\"v2\", l2=\"v3\"}": 1, + "{l1=\"v2\", l2=\"v3\"},{l3=\"v4\"}": 2, } expected[component.Rule] = map[string]int{ - "{l1=\"v2\",l2=\"v3\"}": 2, + "{l1=\"v2\", l2=\"v3\"}": 2, } expected[component.Sidecar] = map[string]int{ fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredStoreAddr[1]): 1, - "{l1=\"v2\",l2=\"v3\"}": 2, + "{l1=\"v2\", l2=\"v3\"}": 2, } expected[component.Store] = map[string]int{ - "": 2, - "{l1=\"v2\",l2=\"v3\"},{l3=\"v4\"}": 3, + "": 2, + "{l1=\"v2\", l2=\"v3\"},{l3=\"v4\"}": 3, } testutil.Equals(t, expected, storeSet.storesMetric.storeNodes) diff --git a/pkg/receive/tsdb_test.go b/pkg/receive/tsdb_test.go index cd9ed3651e..75a80f1b59 100644 --- a/pkg/receive/tsdb_test.go +++ b/pkg/receive/tsdb_test.go @@ -9,8 +9,8 @@ import ( "github.com/go-kit/kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage/tsdb" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -63,7 +63,7 @@ func TestFlushableStorage(t *testing.T) { defer func() { testutil.Ok(t, querier.Close()) }() // Sum the values. - seriesSet, err := querier.Select(labels.NewEqualMatcher("thanos", "flush")) + seriesSet, err := querier.Select(&labels.Matcher{Type: labels.MatchEqual, Name: "thanos", Value: "flush"}) testutil.Ok(t, err) sum := 0.0 for seriesSet.Next() { diff --git a/pkg/rule/rule_test.go b/pkg/rule/rule_test.go index 9c293976a2..5726d2ce36 100644 --- a/pkg/rule/rule_test.go +++ b/pkg/rule/rule_test.go @@ -1,22 +1,87 @@ package thanosrule import ( + "context" "io/ioutil" "os" "path/filepath" "sort" "strings" + "sync" "testing" "time" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/storage" + "github.com/go-kit/kit/log" "github.com/prometheus/prometheus/pkg/rulefmt" "github.com/prometheus/prometheus/rules" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) +type nopAppendable struct{} + +func (n nopAppendable) Add(l labels.Labels, t int64, v float64) (uint64, error) { return 0, nil } +func (n nopAppendable) AddFast(l labels.Labels, ref uint64, t int64, v float64) error { return nil } +func (n nopAppendable) Commit() error { return nil } +func (n nopAppendable) Rollback() error { return nil } +func (n nopAppendable) Appender() (storage.Appender, error) { return n, nil } + +// Regression test against https://github.com/thanos-io/thanos/issues/1779. +func TestRun(t *testing.T) { + dir, err := ioutil.TempDir("", "test_rule_run") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() + + testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "rule.yaml"), []byte(` +groups: +- name: "rule with subquery" + partial_response_strategy: "warn" + rules: + - record: "test" + expr: "rate(some_metric[1h:5m] offset 1d)" +`), os.ModePerm)) + + var ( + queryDone = make(chan struct{}) + queryOnce sync.Once + query string + ) + opts := rules.ManagerOptions{ + Logger: log.NewLogfmtLogger(os.Stderr), + Context: context.Background(), + QueryFunc: func(ctx context.Context, q string, t time.Time) (vectors promql.Vector, e error) { + queryOnce.Do(func() { + query = q + close(queryDone) + }) + return promql.Vector{}, nil + }, + Appendable: nopAppendable{}, + } + thanosRuleMgr := NewManager(dir) + ruleMgr := rules.NewManager(&opts) + thanosRuleMgr.SetRuleManager(storepb.PartialResponseStrategy_ABORT, ruleMgr) + thanosRuleMgr.SetRuleManager(storepb.PartialResponseStrategy_WARN, ruleMgr) + + testutil.Ok(t, thanosRuleMgr.Update(10*time.Second, []string{path.Join(dir, "rule.yaml")})) + + ruleMgr.Run() + defer ruleMgr.Stop() + + select { + case <-time.After(2 * time.Minute): + t.Fatal("timeout while waiting on rule manager query evaluation") + case <-queryDone: + } + + testutil.Equals(t, "rate(some_metric[1h:5m] offset 1d)", query) +} + func TestUpdate(t *testing.T) { dir, err := ioutil.TempDir("", "test_rule_rule_groups") testutil.Ok(t, err) @@ -167,7 +232,7 @@ func TestRuleGroupMarshalYAML(t *testing.T) { - name: something2 rules: - alert: some - expr: up + expr: rate(some_metric[1h:5m] offset 1d) partial_response_strategy: ABORT ` @@ -191,7 +256,7 @@ func TestRuleGroupMarshalYAML(t *testing.T) { Rules: []rulefmt.Rule{ { Alert: "some", - Expr: "up", + Expr: "rate(some_metric[1h:5m] offset 1d)", }, }, }, diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index 3f4267e95d..f06b4478c0 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -17,9 +17,9 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/fileutil" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/objstore" @@ -208,7 +208,7 @@ func (c *lazyOverlapChecker) sync(ctx context.Context) error { return err } - if !labels.FromMap(m.Thanos.Labels).Equals(c.labels()) { + if !labels.Equal(labels.FromMap(m.Thanos.Labels), c.labels()) { return nil } diff --git a/pkg/shipper/shipper_e2e_test.go b/pkg/shipper/shipper_e2e_test.go index 46f6b9d77d..362f27f583 100644 --- a/pkg/shipper/shipper_e2e_test.go +++ b/pkg/shipper/shipper_e2e_test.go @@ -16,9 +16,9 @@ import ( "github.com/go-kit/kit/log" "github.com/oklog/ulid" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/objstore" diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 0c924fe376..e3b65f81b1 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -21,13 +21,12 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - promlabels "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact/downsample" @@ -359,7 +358,7 @@ func (s *BucketStore) SyncBlocks(ctx context.Context) error { // Check for block labels by relabeling. // If output is empty, the block will be dropped. - if processedLabels := relabel.Process(promlabels.FromMap(meta.Thanos.Labels), s.relabelConfig...); processedLabels == nil { + if processedLabels := relabel.Process(labels.FromMap(meta.Thanos.Labels), s.relabelConfig...); processedLabels == nil { level.Debug(s.logger).Log("msg", "ignoring block (drop in relabeling)", "block", id) return os.RemoveAll(bdir) } @@ -644,7 +643,7 @@ func blockSeries( extLset map[string]string, indexr *bucketIndexReader, chunkr *bucketChunkReader, - matchers []labels.Matcher, + matchers []*labels.Matcher, req *storepb.SeriesRequest, samplesLimiter *Limiter, ) (storepb.SeriesSet, *queryStats, error) { @@ -1069,7 +1068,7 @@ func newBucketBlockSet(lset labels.Labels) *bucketBlockSet { } func (s *bucketBlockSet) add(b *bucketBlock) error { - if !s.labels.Equals(labels.FromMap(b.meta.Thanos.Labels)) { + if !labels.Equal(s.labels, labels.FromMap(b.meta.Thanos.Labels)) { return errors.New("block's label set does not match set") } s.mtx.Lock() @@ -1153,11 +1152,11 @@ func (s *bucketBlockSet) getFor(mint, maxt, maxResolutionMillis int64) (bs []*bu // labelMatchers verifies whether the block set matches the given matchers and returns a new // set of matchers that is equivalent when querying data within the block. -func (s *bucketBlockSet) labelMatchers(matchers ...labels.Matcher) ([]labels.Matcher, bool) { - res := make([]labels.Matcher, 0, len(matchers)) +func (s *bucketBlockSet) labelMatchers(matchers ...*labels.Matcher) ([]*labels.Matcher, bool) { + res := make([]*labels.Matcher, 0, len(matchers)) for _, m := range matchers { - v := s.labels.Get(m.Name()) + v := s.labels.Get(m.Name) if v == "" { res = append(res, m) continue @@ -1397,7 +1396,7 @@ func (r *bucketIndexReader) lookupSymbol(o uint32) (string, error) { // Reminder: A posting is a reference (represented as a uint64) to a series reference, which in turn points to the first // chunk where the series contains the matching label-value pair for a given block of data. Postings can be fetched by // single label name=value. -func (r *bucketIndexReader) ExpandedPostings(ms []labels.Matcher) ([]uint64, error) { +func (r *bucketIndexReader) ExpandedPostings(ms []*labels.Matcher) ([]uint64, error) { var postingGroups []*postingGroup // NOTE: Derived from tsdb.PostingsForMatchers. @@ -1478,7 +1477,7 @@ func allWithout(p []index.Postings) index.Postings { } // NOTE: Derived from tsdb.postingsForMatcher. index.Merge is equivalent to map duplication. -func toPostingGroup(lvalsFn func(name string) []string, m labels.Matcher) *postingGroup { +func toPostingGroup(lvalsFn func(name string) []string, m *labels.Matcher) *postingGroup { var matchingLabels labels.Labels // If the matcher selects an empty value, it selects all the series which don't @@ -1488,9 +1487,9 @@ func toPostingGroup(lvalsFn func(name string) []string, m labels.Matcher) *posti allName, allValue := index.AllPostingsKey() matchingLabels = append(matchingLabels, labels.Label{Name: allName, Value: allValue}) - for _, val := range lvalsFn(m.Name()) { + for _, val := range lvalsFn(m.Name) { if !m.Matches(val) { - matchingLabels = append(matchingLabels, labels.Label{Name: m.Name(), Value: val}) + matchingLabels = append(matchingLabels, labels.Label{Name: m.Name, Value: val}) } } @@ -1505,13 +1504,13 @@ func toPostingGroup(lvalsFn func(name string) []string, m labels.Matcher) *posti } // Fast-path for equal matching. - if em, ok := m.(*labels.EqualMatcher); ok { - return newPostingGroup(labels.Labels{{Name: em.Name(), Value: em.Value()}}, merge) + if m.Type == labels.MatchEqual { + return newPostingGroup(labels.Labels{{Name: m.Name, Value: m.Value}}, merge) } - for _, val := range lvalsFn(m.Name()) { + for _, val := range lvalsFn(m.Name) { if m.Matches(val) { - matchingLabels = append(matchingLabels, labels.Label{Name: m.Name(), Value: val}) + matchingLabels = append(matchingLabels, labels.Label{Name: m.Name, Value: val}) } } diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index f6eecd4cd9..d33f530e9b 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -10,9 +10,9 @@ import ( "github.com/go-kit/kit/log" "github.com/oklog/ulid" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/pkg/timestamp" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/model" diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 7f2c2cebac..8586036f6b 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -20,9 +20,9 @@ import ( "github.com/leanovate/gopter/prop" "github.com/oklog/ulid" prommodel "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/pkg/timestamp" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact/downsample" @@ -301,37 +301,37 @@ func TestBucketBlockSet_labelMatchers(t *testing.T) { set := newBucketBlockSet(labels.FromStrings("a", "b", "c", "d")) cases := []struct { - in []labels.Matcher - res []labels.Matcher + in []*labels.Matcher + res []*labels.Matcher match bool }{ { - in: []labels.Matcher{}, - res: []labels.Matcher{}, + in: []*labels.Matcher{}, + res: []*labels.Matcher{}, match: true, }, { - in: []labels.Matcher{ - labels.NewEqualMatcher("a", "b"), - labels.NewEqualMatcher("c", "d"), + in: []*labels.Matcher{ + {Type: labels.MatchEqual, Name: "a", Value: "b"}, + {Type: labels.MatchEqual, Name: "c", Value: "d"}, }, - res: []labels.Matcher{}, + res: []*labels.Matcher{}, match: true, }, { - in: []labels.Matcher{ - labels.NewEqualMatcher("a", "b"), - labels.NewEqualMatcher("c", "b"), + in: []*labels.Matcher{ + {Type: labels.MatchEqual, Name: "a", Value: "b"}, + {Type: labels.MatchEqual, Name: "c", Value: "b"}, }, match: false, }, { - in: []labels.Matcher{ - labels.NewEqualMatcher("a", "b"), - labels.NewEqualMatcher("e", "f"), + in: []*labels.Matcher{ + {Type: labels.MatchEqual, Name: "a", Value: "b"}, + {Type: labels.MatchEqual, Name: "e", Value: "f"}, }, - res: []labels.Matcher{ - labels.NewEqualMatcher("e", "f"), + res: []*labels.Matcher{ + {Type: labels.MatchEqual, Name: "e", Value: "f"}, }, match: true, }, @@ -339,20 +339,20 @@ func TestBucketBlockSet_labelMatchers(t *testing.T) { // We want to provide explicit tests that says when Thanos supports its and when not. We don't support it here in // external labelset level. { - in: []labels.Matcher{ - labels.Not(labels.NewEqualMatcher("", "x")), + in: []*labels.Matcher{ + {Type: labels.MatchNotEqual, Name: "", Value: "x"}, }, - res: []labels.Matcher{ - labels.Not(labels.NewEqualMatcher("", "x")), + res: []*labels.Matcher{ + {Type: labels.MatchNotEqual, Name: "", Value: "x"}, }, match: true, }, { - in: []labels.Matcher{ - labels.Not(labels.NewEqualMatcher("", "d")), + in: []*labels.Matcher{ + {Type: labels.MatchNotEqual, Name: "", Value: "d"}, }, - res: []labels.Matcher{ - labels.Not(labels.NewEqualMatcher("", "d")), + res: []*labels.Matcher{ + {Type: labels.MatchNotEqual, Name: "", Value: "d"}, }, match: true, }, diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index 364239277b..ed4f33cb29 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -10,7 +10,7 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" ) const ( diff --git a/pkg/store/cache/cache_test.go b/pkg/store/cache/cache_test.go index eb87d028e5..3351543cfd 100644 --- a/pkg/store/cache/cache_test.go +++ b/pkg/store/cache/cache_test.go @@ -14,7 +14,7 @@ import ( "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/testutil" ) diff --git a/pkg/store/matchers.go b/pkg/store/matchers.go index 437576d603..d9e4c4065b 100644 --- a/pkg/store/matchers.go +++ b/pkg/store/matchers.go @@ -2,32 +2,28 @@ package store import ( "github.com/pkg/errors" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/store/storepb" ) -func translateMatcher(m storepb.LabelMatcher) (labels.Matcher, error) { +func translateMatcher(m storepb.LabelMatcher) (*labels.Matcher, error) { switch m.Type { case storepb.LabelMatcher_EQ: - return labels.NewEqualMatcher(m.Name, m.Value), nil + return labels.NewMatcher(labels.MatchEqual, m.Name, m.Value) case storepb.LabelMatcher_NEQ: - return labels.Not(labels.NewEqualMatcher(m.Name, m.Value)), nil + return labels.NewMatcher(labels.MatchNotEqual, m.Name, m.Value) case storepb.LabelMatcher_RE: - return labels.NewRegexpMatcher(m.Name, "^(?:"+m.Value+")$") + return labels.NewMatcher(labels.MatchRegexp, m.Name, m.Value) case storepb.LabelMatcher_NRE: - m, err := labels.NewRegexpMatcher(m.Name, "^(?:"+m.Value+")$") - if err != nil { - return nil, err - } - return labels.Not(m), nil + return labels.NewMatcher(labels.MatchNotRegexp, m.Name, m.Value) } return nil, errors.Errorf("unknown label matcher type %d", m.Type) } -func translateMatchers(ms []storepb.LabelMatcher) (res []labels.Matcher, err error) { +func translateMatchers(ms []storepb.LabelMatcher) (res []*labels.Matcher, err error) { for _, m := range ms { r, err := translateMatcher(m) if err != nil { diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 3102fb3465..6b9be0a6a9 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -21,10 +21,10 @@ import ( "github.com/golang/snappy" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage/remote" "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index 609ae11763..bb44ab12d7 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -9,11 +9,11 @@ import ( "time" "github.com/fortytw2/leaktest" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index cb14f06fb2..1817007da7 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -15,7 +15,7 @@ import ( grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/strutil" diff --git a/pkg/store/proxy_test.go b/pkg/store/proxy_test.go index 5aaa1a51de..cc006a4a8e 100644 --- a/pkg/store/proxy_test.go +++ b/pkg/store/proxy_test.go @@ -14,7 +14,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" - tlabels "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" @@ -73,7 +72,7 @@ func TestProxyStore_Series(t *testing.T) { for _, tc := range []struct { title string storeAPIs []Client - selectorLabels tlabels.Labels + selectorLabels labels.Labels req *storepb.SeriesRequest @@ -195,7 +194,7 @@ func TestProxyStore_Series(t *testing.T) { maxTime: 300, }, }, - selectorLabels: tlabels.FromStrings("ext", "2"), + selectorLabels: labels.FromStrings("ext", "2"), req: &storepb.SeriesRequest{ MinTime: 1, MaxTime: 300, @@ -449,7 +448,7 @@ func TestProxyStore_SeriesSlowStores(t *testing.T) { for _, tc := range []struct { title string storeAPIs []Client - selectorLabels tlabels.Labels + selectorLabels labels.Labels req *storepb.SeriesRequest @@ -638,7 +637,7 @@ func TestProxyStore_Series_RegressionFillResponseChannel(t *testing.T) { q := NewProxyStore(nil, func() []Client { return cls }, component.Query, - tlabels.FromStrings("fed", "a"), + labels.FromStrings("fed", "a"), 0*time.Second, ) @@ -1064,7 +1063,7 @@ func storeSeriesResponse(t testing.TB, lset labels.Labels, smplChunks ...[]sampl func TestMergeLabels(t *testing.T) { ls := []storepb.Label{{Name: "a", Value: "b"}, {Name: "b", Value: "c"}} - selector := tlabels.Labels{{Name: "a", Value: "c"}, {Name: "c", Value: "d"}} + selector := labels.Labels{{Name: "a", Value: "c"}, {Name: "c", Value: "d"}} expected := labels.Labels{{Name: "a", Value: "c"}, {Name: "b", Value: "c"}, {Name: "c", Value: "d"}} res := mergeLabels(ls, selector) diff --git a/pkg/store/tsdb.go b/pkg/store/tsdb.go index 6ce8982f95..db818d67cd 100644 --- a/pkg/store/tsdb.go +++ b/pkg/store/tsdb.go @@ -8,9 +8,9 @@ import ( "github.com/go-kit/kit/log" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" diff --git a/pkg/store/tsdb_test.go b/pkg/store/tsdb_test.go index 2e853361d8..6819f74c2f 100644 --- a/pkg/store/tsdb_test.go +++ b/pkg/store/tsdb_test.go @@ -7,7 +7,7 @@ import ( "time" "github.com/fortytw2/leaktest" - "github.com/prometheus/prometheus/tsdb/labels" + "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" diff --git a/pkg/testutil/prometheus.go b/pkg/testutil/prometheus.go index b89aac77f2..209f4e5175 100644 --- a/pkg/testutil/prometheus.go +++ b/pkg/testutil/prometheus.go @@ -19,8 +19,8 @@ import ( "github.com/go-kit/kit/log" "github.com/oklog/ulid" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/runutil" "golang.org/x/sync/errgroup" diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 5cf5078afe..b3df6d1f8f 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -11,8 +11,8 @@ import ( "github.com/go-kit/kit/log" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" - "github.com/prometheus/prometheus/tsdb/labels" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/s3" From 82c005d9896c2e21b615074779bfba6304f2a332 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 26 Nov 2019 13:20:09 -0800 Subject: [PATCH 067/257] Cut release v0.9.0-rc.0 (#1793) Signed-off-by: Bartek Plotka --- CHANGELOG.md | 38 +++++++++++++++++++++++--------------- VERSION | 2 +- pkg/rule/rule_test.go | 11 +++++------ 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b27baed89..05359a22f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,36 +11,44 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased +## [v0.9.0-rc.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0-rc.0) - 2019.11.26 + ### Added - [#1678](https://github.com/thanos-io/thanos/pull/1678) Add Lightstep as a tracing provider. - [#1687](https://github.com/thanos-io/thanos/pull/1687) Add a new `--grpc-grace-period` CLI option to components which serve gRPC to set how long to wait until gRPC Server shuts down. -- [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. +- [#1660](https://github.com/thanos-io/thanos/pull/1660) Sidecar: Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up. - [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss) for further information. - [#1680](https://github.com/thanos-io/thanos/pull/1680) Add a new `--http-grace-period` CLI option to components which serve HTTP to set how long to wait until HTTP Server shuts down. -- [#1712](https://github.com/thanos-io/thanos/pull/1712) Rename flag on bucket web component from `--listen` to `--http-address` to match other components. -- [#1733](https://github.com/thanos-io/thanos/pull/1733) New metric `thanos_compactor_iterations_total` on Thanos Compactor which shows the number of successful iterations. -- [#1758](https://github.com/thanos-io/thanos/pull/1758) `thanos bucket web` now supports `--web.external-prefix` for proxying on a subpath. -- [#1770](https://github.com/thanos-io/thanos/pull/1770) Add `--web.prefix-header` flags to allow for bucket UI to be accessible behind a reverse proxy. +- [#1712](https://github.com/thanos-io/thanos/pull/1712) Bucket: Rename flag on bucket web component from `--listen` to `--http-address` to match other components. +- [#1733](https://github.com/thanos-io/thanos/pull/1733) Compactor: New metric `thanos_compactor_iterations_total` on Thanos Compactor which shows the number of successful iterations. +- [#1758](https://github.com/thanos-io/thanos/pull/1758) Bucket: `thanos bucket web` now supports `--web.external-prefix` for proxying on a subpath. +- [#1770](https://github.com/thanos-io/thanos/pull/1770) Bucket: Add `--web.prefix-header` flags to allow for bucket UI to be accessible behind a reverse proxy. +- [#1668](https://github.com/thanos-io/thanos/pull/1668) Receiver: Added TLS options for both server and client remote write. ### Fixed -- [#1656](https://github.com/thanos-io/thanos/pull/1656) Thanos Store now starts metric and status probe HTTP server earlier in its start-up sequence. `/-/healthy` endpoint now starts to respond with success earlier. `/metrics` endpoint starts serving metrics earlier as well. Make sure to point your readiness probes to the `/-/ready` endpoint rather than `/metrics`. -- [#1669](https://github.com/thanos-io/thanos/pull/1669) Fixed store sharding. Now it does not load excluded meta.jsons and load/fetch index-cache.json files. -- [#1670](https://github.com/thanos-io/thanos/pull/1670) Fixed un-ordered blocks upload. Sidecar now uploads the oldest blocks first. -- [#1568](https://github.com/thanos-io/thanos/pull/1709) Thanos Store now retains the first raw value of a chunk during downsampling to avoid losing some counter resets that occur on an aggregation boundary. -- [#1773](https://github.com/thanos-io/thanos/pull/1773) Thanos Ruler: fixed the /api/v1/rules endpoint that returned 500 status code with `failed to assert type of rule ...` message. -- [#1770](https://github.com/thanos-io/thanos/pull/1770) Fix `--web.external-prefix` 404s for static resources. -- [#1785](https://github.com/thanos-io/thanos/pull/1785) Thanos Ruler: the /api/v1/rules endpoints now returns the original rule filenames. -- [#1791](https://github.com/thanos-io/thanos/pull/1791) Thanos Ruler now supports identical rule filenames in different directories. +- [#1656](https://github.com/thanos-io/thanos/pull/1656) Store Gateway: Store now starts metric and status probe HTTP server earlier in its start-up sequence. `/-/healthy` endpoint now starts to respond with success earlier. `/metrics` endpoint starts serving metrics earlier as well. Make sure to point your readiness probes to the `/-/ready` endpoint rather than `/metrics`. +- [#1669](https://github.com/thanos-io/thanos/pull/1669) Store Gateway: Fixed store sharding. Now it does not load excluded meta.jsons and load/fetch index-cache.json files. +- [#1670](https://github.com/thanos-io/thanos/pull/1670) Sidecar: Fixed un-ordered blocks upload. Sidecar now uploads the oldest blocks first. +- [#1568](https://github.com/thanos-io/thanos/pull/1709) Store Gateway: Store now retains the first raw value of a chunk during downsampling to avoid losing some counter resets that occur on an aggregation boundary. +- [#1751](https://github.com/thanos-io/thanos/pull/1751) Querier: Fixed labels for StoreUI +- [#1773](https://github.com/thanos-io/thanos/pull/1773) Ruler: Fixed the /api/v1/rules endpoint that returned 500 status code with `failed to assert type of rule ...` message. +- [#1770](https://github.com/thanos-io/thanos/pull/1770) Querier: Fixed `--web.external-prefix` 404s for static resources. +- [#1785](https://github.com/thanos-io/thanos/pull/1785) Ruler: The /api/v1/rules endpoints now returns the original rule filenames. +- [#1791](https://github.com/thanos-io/thanos/pull/1791) Ruler: Ruler now supports identical rule filenames in different directories. +- [#1562](https://github.com/thanos-io/thanos/pull/1562) Querier: Downsampling option now carries through URL. +- [#1675](https://github.com/thanos-io/thanos/pull/1675) Querier: Reduced resource usage while using certain queries like `offset`. +- [#1725](https://github.com/thanos-io/thanos/pull/1725) & [#1718](https://github.com/thanos-io/thanos/pull/1718) Store Gateway: Per request memory improvements. ### Changed -- [#1666](https://github.com/thanos-io/thanos/pull/1666) `thanos_compact_group_compactions_total` now counts block compactions, so operations that resulted in a compacted block. The old behaviour +- [#1666](https://github.com/thanos-io/thanos/pull/1666) Compact: `thanos_compact_group_compactions_total` now counts block compactions, so operations that resulted in a compacted block. The old behaviour is now exposed by new metric: `thanos_compact_group_compaction_runs_started_total` and `thanos_compact_group_compaction_runs_completed_total` which counts compaction runs overall. - [#1748](https://github.com/thanos-io/thanos/pull/1748) Updated all dependencies. - [#1694](https://github.com/thanos-io/thanos/pull/1694) `prober_ready` and `prober_healthy` metrics are removed, for sake of `status`. Now `status` exposes same metric with a label, `check`. `check` can have "healty" or "ready" depending on status of the probe. -- [#1790](https://github.com/thanos-io/thanos/pull/1790) Fixes subqueries support for ruler. +- [#1790](https://github.com/thanos-io/thanos/pull/1790) Ruler: Fixes subqueries support for ruler. +- [#1769](https://github.com/thanos-io/thanos/pull/1769) & [#1545](https://github.com/thanos-io/thanos/pull/1545) Adjusted most of the metrics histogram buckets. ## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14 diff --git a/VERSION b/VERSION index d182dc9160..1091a9206e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8.1-dev +0.9.0-rc.0 diff --git a/pkg/rule/rule_test.go b/pkg/rule/rule_test.go index 5726d2ce36..0e1cc9ad0b 100644 --- a/pkg/rule/rule_test.go +++ b/pkg/rule/rule_test.go @@ -11,13 +11,12 @@ import ( "testing" "time" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/storage" - "github.com/go-kit/kit/log" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" + "github.com/prometheus/prometheus/storage" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" "gopkg.in/yaml.v2" @@ -37,7 +36,7 @@ func TestRun(t *testing.T) { testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "rule.yaml"), []byte(` + testutil.Ok(t, ioutil.WriteFile(filepath.Join(dir, "rule.yaml"), []byte(` groups: - name: "rule with subquery" partial_response_strategy: "warn" @@ -68,7 +67,7 @@ groups: thanosRuleMgr.SetRuleManager(storepb.PartialResponseStrategy_ABORT, ruleMgr) thanosRuleMgr.SetRuleManager(storepb.PartialResponseStrategy_WARN, ruleMgr) - testutil.Ok(t, thanosRuleMgr.Update(10*time.Second, []string{path.Join(dir, "rule.yaml")})) + testutil.Ok(t, thanosRuleMgr.Update(10*time.Second, []string{filepath.Join(dir, "rule.yaml")})) ruleMgr.Run() defer ruleMgr.Stop() From 46bb3c4a8b6767f811a58205b63150366256281a Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Wed, 27 Nov 2019 17:55:19 +0800 Subject: [PATCH 068/257] delete broken link (#1799) Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/components/compact.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/components/compact.md b/docs/components/compact.md index 4a710a6773..a6472c7c99 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -50,8 +50,8 @@ In fact, downsampling doesn't save you any space but instead it adds 2 more bloc ## Groups -The compactor groups blocks using the [external_labels](https://thanos.io/getting-started.md/#external-labels) added by the -Prometheus who produced the block. The labels must be both _unique_ and _persistent_ across different Prometheus instances. +The compactor groups blocks using the external_labels added by the Prometheus who produced the block. +The labels must be both _unique_ and _persistent_ across different Prometheus instances. By _unique_, we mean that the set of labels in a Prometheus instance must be different from all other sets of labels of your Prometheus instances, so that the compactor will be able to group blocks by Prometheus instance. From 4e06e2e6c2effa9d86b0e80d1b431e74d5276469 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Wed, 27 Nov 2019 18:12:28 +0800 Subject: [PATCH 069/257] add --web.enable-admin-api comment (#1800) * add --web.enable-admin-api comment Signed-off-by: Xiang Dai <764524258@qq.com> * remove white notes Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/components/sidecar.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 4dfc4282b2..2d735061f3 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -8,12 +8,12 @@ menu: components The sidecar component of Thanos gets deployed along with a Prometheus instance. This allows sidecar to optionally upload metrics to object storage and allow [Queriers](./query.md) to query Prometheus data with common, efficient StoreAPI. -In details: +In details: * It implements Thanos' Store API on top of Prometheus' remote-read API. This allows [Queriers](./query.md) to treat Prometheus servers as yet another source of time series data without directly talking to its APIs. * Optionally, the sidecar uploads TSDB blocks to an object storage bucket as Prometheus produces them every 2 hours. This allows Prometheus servers to be run with relatively low retention while their historic data is made durable and queryable via object storage. - NOTE: This still does NOT mean that Prometheus can be fully stateless, because if it crashes and restarts you will lose ~2 hours of metrics, so persistent disk for Prometheus is highly recommended. The closest to stateless you can get is using remote write (which Thanos experimentally supports, see [this](../proposals/201812_thanos-remote-receive.md). Remote write has other risks and consequences, and still if crashed you loose in positive case seconds of metrics data, so persistent disk is recommended in all cases. + NOTE: This still does NOT mean that Prometheus can be fully stateless, because if it crashes and restarts you will lose ~2 hours of metrics, so persistent disk for Prometheus is highly recommended. The closest to stateless you can get is using remote write (which Thanos experimentally supports, see [this](../proposals/201812_thanos-remote-receive.md). Remote write has other risks and consequences, and still if crashed you loose in positive case seconds of metrics data, so persistent disk is recommended in all cases. * Optionally Thanos sidecar is able to watch Prometheus rules and configuration, decompress and substitute environment variables if needed and ping Prometheus to reload them. Read more about this in [here](./sidecar.md#reloader-configuration) @@ -22,15 +22,16 @@ Prometheus servers connected to the Thanos cluster via the sidecar are subject t * The recommended Prometheus version is 2.2.1 or greater (including newest releases). This is due to Prometheus instability in previous versions as well as lack of `flags` endpoint. * (!) The Prometheus `external_labels` section of the Prometheus configuration file has unique labels in the overall Thanos system. Those external labels will be used by the sidecar and then Thanos in many places: - + * [Querier](./query.md) to filter out store APIs to touch during query requests. * Many object storage readers like [compactor](./compact.md) and [store gateway](./store.md) which groups the blocks by Prometheus source. Each produced TSDB block by Prometheus is labelled with external label by sidecar before upload to object storage. - + +* The `--web.enable-admin-api` flag is enabled to support sidecar to get metadata from Prometheus like external labels. * The `--web.enable-lifecycle` flag is enabled if you want to use sidecar reloading features (`--reload.*` flags). If you choose to use the sidecar to also upload to object storage: -* The `--storage.tsdb.min-block-duration` and `--storage.tsdb.max-block-duration` must be set to equal values to disable local compaction on order to use Thanos sidecar upload, otherwise leave local compaction on if sidecar just exposes StoreAPI and your retention is normal. The default of `2h` is recommended. +* The `--storage.tsdb.min-block-duration` and `--storage.tsdb.max-block-duration` must be set to equal values to disable local compaction on order to use Thanos sidecar upload, otherwise leave local compaction on if sidecar just exposes StoreAPI and your retention is normal. The default of `2h` is recommended. Mentioned parameters set to equal values disable the internal Prometheus compaction, which is needed to avoid the uploaded data corruption when Thanos compactor does its job, this is critical for data consistency and should not be ignored if you plan to use Thanos compactor. Even though you set mentioned parameters equal, you might observe Prometheus internal metric `prometheus_tsdb_compactions_total` being incremented, don't be confused by that: Prometheus writes initial head block to filesytem via its internal compaction mechanism, but if you have followed recommendations - data won't be modified by Prometheus before the sidecar uploads it. Thanos sidecar will also check sanity of the flags set to Prometheus on the startup and log errors or warning if they have been configured improperly (#838). * The retention is recommended to not be lower than three times the min block duration, so 6 hours. This achieves resilience in the face of connectivity issues to the object storage since all local data will remain available within the Thanos cluster. If connectivity gets restored the backlog of blocks gets uploaded to the object storage. From 56c1ae6be2fb9a7b7c1d2846ffa1ed3a0ea466e4 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Wed, 27 Nov 2019 09:06:55 -0800 Subject: [PATCH 070/257] Fixed cross build and added cross build per PR on GH actions. (#1806) * Added cross build per PR on GH actions. Signed-off-by: Bartek Plotka * Further fix. Signed-off-by: Bartek Plotka * Left just cross build check. Signed-off-by: Bartek Plotka --- .github/workflows/ci.yaml | 21 +++++++++++++++++++++ go.mod | 1 - go.sum | 27 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..a106831af9 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,21 @@ +name: CI +on: [pull_request] + +jobs: + cross-build-check: + name: Cross build check + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + + - name: Get dependencies + run: make deps + + - name: Cross build check + run: make crossbuild diff --git a/go.mod b/go.mod index 3a91fe1b1b..141106d1af 100644 --- a/go.mod +++ b/go.mod @@ -112,7 +112,6 @@ require ( replace ( // Mitigation for: https://github.com/Azure/go-autorest/issues/414 github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.3.0+incompatible - golang.org/x/sys => golang.org/x/sys v0.0.0-20190412213103-97732733099d // v0.0.0-20190425145619-16072639606e (multiple-value "golang.org/x/sys/windows".GetCurrentProcess() in single-value context) Required to build properly on windows for github.com/elastic/go-sysinfo. k8s.io/api => k8s.io/api v0.0.0-20190620084959-7cf5895f2711 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190620085554-14e95df34f1f k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719 diff --git a/go.sum b/go.sum index b8c2309441..9964e1ec60 100644 --- a/go.sum +++ b/go.sum @@ -587,8 +587,35 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 h1:dHtDnRWQtSx0Hjq9kvKFpBh9uPPKfQN70NZZmvssGwk= +golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180805044716-cb6730876b98/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 7345ab83cea9d44db8e4306f33d242a3d8df35a8 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 28 Nov 2019 18:12:31 +0800 Subject: [PATCH 071/257] update prometheus version comment (#1809) Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/quick-tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quick-tutorial.md b/docs/quick-tutorial.md index 479ba2ab83..d37f71ba9b 100644 --- a/docs/quick-tutorial.md +++ b/docs/quick-tutorial.md @@ -20,7 +20,7 @@ Prometheus always stays as integral foundation for *collecting metrics* and aler Thanos bases itself on vanilla [Prometheus](https://prometheus.io/) (v2.2.1+). We plan to support *all* Prometheus version beyond this version. -NOTE: It is highly recommended to use Prometheus 2.13 (available in next Prometheus release) due to Prometheus remote read improvements. +NOTE: It is highly recommended to use Prometheus v2.13+ due to Prometheus remote read improvements. Always make sure to run Prometheus as recommended by Prometheus team, so: From 7b65bf53b00057937f4bd6268b82463c8834c7ad Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 28 Nov 2019 18:13:26 +0800 Subject: [PATCH 072/257] update image tag (#1797) Signed-off-by: Xiang Dai <764524258@qq.com> --- tutorials/kubernetes-helm/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutorials/kubernetes-helm/README.md b/tutorials/kubernetes-helm/README.md index d5b1d2d23b..3a9cb31258 100644 --- a/tutorials/kubernetes-helm/README.md +++ b/tutorials/kubernetes-helm/README.md @@ -64,7 +64,8 @@ server: sidecarContainers: - name: thanos-sidecar # Always use explicit image tags (release or master--sha) instead of ambigous `latest` or `master`. - image: improbable/thanos:v0.3.2 + # Check https://quay.io/repository/thanos/thanos?tab=tags to get latest tag. + image: quay.io/thanos/thanos:master-2019-11-26-82c005d9 resources: requests: memory: "4Gi" @@ -80,7 +81,6 @@ server: - "--log.level=debug" - "--tsdb.path=/data/" - "--prometheus.url=http://127.0.0.1:9090" - - "--cluster.disable" - "--objstore.config={type: GCS, config: {bucket: BUCKET_REPLACE_ME}}" - "--reloader.config-file=/etc/prometheus-config/prometheus.yml" - "--reloader.config-envsubst-file=/etc/prometheus-shared/prometheus.yml" From 6de3778e9afb4c95274bb933670ed77234662e5a Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 28 Nov 2019 21:09:36 +0800 Subject: [PATCH 073/257] Katacoda: Add external url & Updated docker images (#1802) * add external url Signed-off-by: Xiang Dai <764524258@qq.com> * update thanos image tag Signed-off-by: Xiang Dai <764524258@qq.com> --- tutorials/katacoda/thanos/1-globalview/courseBase.sh | 4 ++-- tutorials/katacoda/thanos/1-globalview/step1.md | 9 ++++++--- tutorials/katacoda/thanos/1-globalview/step2.md | 8 ++++---- tutorials/katacoda/thanos/1-globalview/step3.md | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tutorials/katacoda/thanos/1-globalview/courseBase.sh b/tutorials/katacoda/thanos/1-globalview/courseBase.sh index ae69695eb2..196c8b5b9f 100644 --- a/tutorials/katacoda/thanos/1-globalview/courseBase.sh +++ b/tutorials/katacoda/thanos/1-globalview/courseBase.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash -docker pull quay.io/thanos/prometheus:v2.12.0-rc.0-rr-streaming -docker pull quay.io/thanos/thanos:v0.7.0 \ No newline at end of file +docker pull quay.io/prometheus/prometheus:v2.14.0 +docker pull quay.io/thanos/thanos:v0.9.0-rc.0 diff --git a/tutorials/katacoda/thanos/1-globalview/step1.md b/tutorials/katacoda/thanos/1-globalview/step1.md index 2bf628ef07..8725d423e9 100644 --- a/tutorials/katacoda/thanos/1-globalview/step1.md +++ b/tutorials/katacoda/thanos/1-globalview/step1.md @@ -95,10 +95,11 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_eu1_data:/prometheus \ -u root \ --name prometheus-0-eu1 \ - quay.io/thanos/prometheus:v2.12.0-rc.0-rr-streaming \ + quay.io/prometheus/prometheus:v2.14.0 \ --config.file=/etc/prometheus/prometheus.yml \ --storage.tsdb.path=/prometheus \ --web.listen-address=:9090 \ + --web.external-url=https://[[HOST_SUBDOMAIN]]-9090-[[KATACODA_HOST]].environments.katacoda.com \ --web.enable-lifecycle \ --web.enable-admin-api && echo "Prometheus EU1 started!" ```{{execute}} @@ -113,10 +114,11 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_us1_data:/prometheus \ -u root \ --name prometheus-0-us1 \ - quay.io/thanos/prometheus:v2.12.0-rc.0-rr-streaming \ + quay.io/prometheus/prometheus:v2.14.0 \ --config.file=/etc/prometheus/prometheus.yml \ --storage.tsdb.path=/prometheus \ --web.listen-address=:9091 \ + --web.external-url=https://[[HOST_SUBDOMAIN]]-9091-[[KATACODA_HOST]].environments.katacoda.com \ --web.enable-lifecycle \ --web.enable-admin-api && echo "Prometheus 0 US1 started!" ```{{execute}} @@ -129,10 +131,11 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus1_us1_data:/prometheus \ -u root \ --name prometheus-1-us1 \ - quay.io/thanos/prometheus:v2.12.0-rc.0-rr-streaming \ + quay.io/prometheus/prometheus:v2.14.0 \ --config.file=/etc/prometheus/prometheus.yml \ --storage.tsdb.path=/prometheus \ --web.listen-address=:9092 \ + --web.external-url=https://[[HOST_SUBDOMAIN]]-9092-[[KATACODA_HOST]].environments.katacoda.com \ --web.enable-lifecycle \ --web.enable-admin-api && echo "Prometheus 1 US1 started!" ```{{execute}} diff --git a/tutorials/katacoda/thanos/1-globalview/step2.md b/tutorials/katacoda/thanos/1-globalview/step2.md index 7beef7ef7c..46cf27b08c 100644 --- a/tutorials/katacoda/thanos/1-globalview/step2.md +++ b/tutorials/katacoda/thanos/1-globalview/step2.md @@ -10,7 +10,7 @@ component and can be invoked in a single command. Let's take a look at all the Thanos commands: ``` -docker run --rm quay.io/thanos/thanos:v0.7.0 --help +docker run --rm quay.io/thanos/thanos:v0.9.0-rc.0 --help ```{{execute}} You should see multiple commands that solves different purposes. @@ -53,7 +53,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_eu1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-eu1 \ -u root \ - quay.io/thanos/thanos:v0.7.0 \ + quay.io/thanos/thanos:v0.9.0-rc.0 \ sidecar \ --http-address 0.0.0.0:19090 \ --grpc-address 0.0.0.0:19190 \ @@ -68,7 +68,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.7.0 \ + quay.io/thanos/thanos:v0.9.0-rc.0 \ sidecar \ --http-address 0.0.0.0:19091 \ --grpc-address 0.0.0.0:19191 \ @@ -81,7 +81,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus1_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-1-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.7.0 \ + quay.io/thanos/thanos:v0.9.0-rc.0 \ sidecar \ --http-address 0.0.0.0:19092 \ --grpc-address 0.0.0.0:19192 \ diff --git a/tutorials/katacoda/thanos/1-globalview/step3.md b/tutorials/katacoda/thanos/1-globalview/step3.md index afd9bab5f5..32843c7802 100644 --- a/tutorials/katacoda/thanos/1-globalview/step3.md +++ b/tutorials/katacoda/thanos/1-globalview/step3.md @@ -28,7 +28,7 @@ Click below snippet to start the Querier. ``` docker run -d --net=host --rm \ --name querier \ - quay.io/thanos/thanos:v0.7.0 \ + quay.io/thanos/thanos:v0.9.0-rc.0 \ query \ --http-address 0.0.0.0:29090 \ --query.replica-label replica \ From 6cad4cac17511d17f8987069ffa19ec35481e879 Mon Sep 17 00:00:00 2001 From: Antonio Santos Date: Thu, 28 Nov 2019 14:21:11 +0100 Subject: [PATCH 074/257] Fix typo in promlabels package name (#1805) * Fix typo in package name Signed-off-by: Antonio Santos * Remove redundant import Signed-off-by: Antonio Santos --- pkg/compact/compact.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 37ca4fdf76..277904eeec 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -17,7 +17,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" - promlables "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb" terrors "github.com/prometheus/prometheus/tsdb/errors" @@ -230,7 +229,7 @@ func (c *Syncer) syncMetas(ctx context.Context) error { // Check for block labels by relabeling. // If output is empty, the block will be dropped. - lset := promlables.FromMap(meta.Thanos.Labels) + lset := labels.FromMap(meta.Thanos.Labels) processedLabels := relabel.Process(lset, c.relabelConfig...) if processedLabels == nil { level.Debug(c.logger).Log("msg", "dropping block(drop in relabeling)", "block", id) From 229f978dffca4fda136d455e1553cf8cf08a3c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 30 Nov 2019 10:52:49 +0200 Subject: [PATCH 075/257] objstore: add ObjectSize() implementation (#1792) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * objstore: add initial ObjectSize() implementation Adds a new method ObjectSize(context.Context, string) method to our object storage interface which returns the size of the specified object in uint64 without actually downloading all of it. This is a pre-requisite to one check that we want to add. Add a check to our E2E tests to see if it works. Signed-off-by: Giedrius Statkevičius * objstore: cos/oss: fix linter errors Signed-off-by: Giedrius Statkevičius * objstore: return direct err in ObjectSize() The exact error is needed by the callers so that they could check `IsObjNotFoundErr` correctly. Add a test for this to the E2E tests. Signed-off-by: Giedrius Statkevičius * objstore: azure: return err directly as well Signed-off-by: Giedrius Statkevičius --- pkg/objstore/azure/azure.go | 14 +++++++++++++ pkg/objstore/cos/cos.go | 20 +++++++++++++++++++ pkg/objstore/filesystem/filesystem.go | 10 ++++++++++ pkg/objstore/gcs/gcs.go | 9 +++++++++ pkg/objstore/inmem/inmem.go | 11 ++++++++++ pkg/objstore/objstore.go | 18 +++++++++++++++++ .../objtesting/acceptance_e2e_test.go | 9 +++++++++ pkg/objstore/oss/oss.go | 20 +++++++++++++++++++ pkg/objstore/s3/s3.go | 9 +++++++++ pkg/objstore/swift/swift.go | 10 ++++++++++ 10 files changed, 130 insertions(+) diff --git a/pkg/objstore/azure/azure.go b/pkg/objstore/azure/azure.go index 83f36a81a1..b5ef93ab3d 100644 --- a/pkg/objstore/azure/azure.go +++ b/pkg/objstore/azure/azure.go @@ -228,6 +228,20 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return b.getBlobReader(ctx, name, off, length) } +// ObjectSize returns the size of the specified object. +func (b *Bucket) ObjectSize(ctx context.Context, name string) (uint64, error) { + blobURL, err := getBlobURL(ctx, *b.config, name) + if err != nil { + return 0, errors.Wrapf(err, "cannot get Azure blob URL, blob: %s", name) + } + var props *blob.BlobGetPropertiesResponse + props, err = blobURL.GetProperties(ctx, blob.BlobAccessConditions{}) + if err != nil { + return 0, err + } + return uint64(props.ContentLength()), nil +} + // Exists checks if the given object exists. func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { level.Debug(b.logger).Log("msg", "check if blob exists", "blob", name) diff --git a/pkg/objstore/cos/cos.go b/pkg/objstore/cos/cos.go index 19fe3ceacc..df4fc04ee0 100644 --- a/pkg/objstore/cos/cos.go +++ b/pkg/objstore/cos/cos.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "os" + "strconv" "strings" "testing" @@ -88,6 +89,25 @@ func (b *Bucket) Name() string { return b.name } +// ObjectSize returns the size of the specified object. +func (b *Bucket) ObjectSize(ctx context.Context, name string) (uint64, error) { + resp, err := b.client.Object.Head(ctx, name, nil) + if err != nil { + return 0, err + } + if v, ok := resp.Header["Content-Length"]; ok { + if len(v) == 0 { + return 0, errors.New("content-length header has no values") + } + ret, err := strconv.ParseUint(v[0], 10, 64) + if err != nil { + return 0, errors.Wrap(err, "convert content-length") + } + return ret, nil + } + return 0, errors.New("content-length header not found") +} + // Upload the contents of the reader as an object into the bucket. func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { if _, err := b.client.Object.Put(ctx, name, r, nil); err != nil { diff --git a/pkg/objstore/filesystem/filesystem.go b/pkg/objstore/filesystem/filesystem.go index fc0a11a717..cac855588a 100644 --- a/pkg/objstore/filesystem/filesystem.go +++ b/pkg/objstore/filesystem/filesystem.go @@ -104,6 +104,16 @@ func (r *rangeReaderCloser) Close() error { return r.f.Close() } +// ObjectSize returns the size of the specified object. +func (b *Bucket) ObjectSize(_ context.Context, name string) (uint64, error) { + file := filepath.Join(b.rootDir, name) + st, err := os.Stat(file) + if err != nil { + return 0, errors.Wrapf(err, "stat %s", file) + } + return uint64(st.Size()), nil +} + // GetRange returns a new range reader for the given object name and range. func (b *Bucket) GetRange(_ context.Context, name string, off, length int64) (io.ReadCloser, error) { if name == "" { diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index b3ad6ef350..533659dc28 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -122,6 +122,15 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return b.bkt.Object(name).NewRangeReader(ctx, off, length) } +// ObjectSize returns the size of the specified object. +func (b *Bucket) ObjectSize(ctx context.Context, name string) (uint64, error) { + obj, err := b.bkt.Object(name).Attrs(ctx) + if err != nil { + return 0, err + } + return uint64(obj.Size), nil +} + // Handle returns the underlying GCS bucket handle. // Used for testing purposes (we return handle, so it is not instrumented). func (b *Bucket) Handle() *storage.BucketHandle { diff --git a/pkg/objstore/inmem/inmem.go b/pkg/objstore/inmem/inmem.go index 25e3196b96..8b04974272 100644 --- a/pkg/objstore/inmem/inmem.go +++ b/pkg/objstore/inmem/inmem.go @@ -139,6 +139,17 @@ func (b *Bucket) Exists(_ context.Context, name string) (bool, error) { return ok, nil } +// ObjectSize returns the size of the specified object. +func (b *Bucket) ObjectSize(_ context.Context, name string) (uint64, error) { + b.mtx.RLock() + file, ok := b.objects[name] + b.mtx.RUnlock() + if !ok { + return 0, errNotFound + } + return uint64(len(file)), nil +} + // Upload writes the file specified in src to into the memory. func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { b.mtx.Lock() diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index f328019b07..5a70d885dc 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -54,6 +54,9 @@ type BucketReader interface { // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. IsObjNotFoundErr(err error) bool + + // ObjectSize returns the size of the specified object. + ObjectSize(ctx context.Context, name string) (uint64, error) } // UploadDir uploads all files in srcdir to the bucket with into a top-level directory @@ -238,6 +241,21 @@ func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string) return err } +// ObjectSize returns the size of the specified object. +func (b *metricBucket) ObjectSize(ctx context.Context, name string) (uint64, error) { + const op = "objectsize" + b.ops.WithLabelValues(op).Inc() + start := time.Now() + + rc, err := b.bkt.ObjectSize(ctx, name) + if err != nil { + b.opsFailures.WithLabelValues(op).Inc() + return 0, err + } + b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) + return rc, nil +} + func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { const op = "get" b.ops.WithLabelValues(op).Inc() diff --git a/pkg/objstore/objtesting/acceptance_e2e_test.go b/pkg/objstore/objtesting/acceptance_e2e_test.go index ad743bd60e..ce918289f7 100644 --- a/pkg/objstore/objtesting/acceptance_e2e_test.go +++ b/pkg/objstore/objtesting/acceptance_e2e_test.go @@ -31,6 +31,10 @@ func TestObjStore_AcceptanceTest_e2e(t *testing.T) { testutil.Ok(t, err) testutil.Assert(t, !ok, "expected not exits") + _, err = bkt.ObjectSize(ctx, "id1/obj_1.some") + testutil.NotOk(t, err) + testutil.Assert(t, bkt.IsObjNotFoundErr(err), "expected not found error but got %s", err) + // Upload first object. testutil.Ok(t, bkt.Upload(ctx, "id1/obj_1.some", strings.NewReader("@test-data@"))) @@ -42,6 +46,11 @@ func TestObjStore_AcceptanceTest_e2e(t *testing.T) { testutil.Ok(t, err) testutil.Equals(t, "@test-data@", string(content)) + // Check if we can get the correct size. + sz, err := bkt.ObjectSize(ctx, "id1/obj_1.some") + testutil.Ok(t, err) + testutil.Assert(t, sz == 11, "expected size to be equal to 11") + rc2, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, 3) testutil.Ok(t, err) defer func() { testutil.Ok(t, rc2.Close()) }() diff --git a/pkg/objstore/oss/oss.go b/pkg/objstore/oss/oss.go index 6f7b6320d3..e4b2b14224 100644 --- a/pkg/objstore/oss/oss.go +++ b/pkg/objstore/oss/oss.go @@ -137,6 +137,26 @@ func (b *Bucket) Delete(ctx context.Context, name string) error { return nil } +// ObjectSize returns the size of the specified object. +func (b *Bucket) ObjectSize(ctx context.Context, name string) (uint64, error) { + // https://github.com/aliyun/aliyun-oss-go-sdk/blob/cee409f5b4d75d7ad077cacb7e6f4590a7f2e172/oss/bucket.go#L668 + m, err := b.bucket.GetObjectMeta(name) + if err != nil { + return 0, err + } + if v, ok := m["Content-Length"]; ok { + if len(v) == 0 { + return 0, errors.New("content-length header has no values") + } + ret, err := strconv.ParseUint(v[0], 10, 64) + if err != nil { + return 0, errors.Wrap(err, "convert content-length") + } + return ret, nil + } + return 0, errors.New("content-length header not found") +} + // NewBucket returns a new Bucket using the provided oss config values. func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { var config Config diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 7bbb96d585..2ca9d187c4 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -338,6 +338,15 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { return nil } +// ObjectSize returns the size of the specified object. +func (b *Bucket) ObjectSize(ctx context.Context, name string) (uint64, error) { + objInfo, err := b.client.StatObject(b.name, name, minio.StatObjectOptions{}) + if err != nil { + return 0, err + } + return uint64(objInfo.Size), nil +} + // Delete removes the object with the given name. func (b *Bucket) Delete(ctx context.Context, name string) error { return b.client.RemoveObject(b.name, name) diff --git a/pkg/objstore/swift/swift.go b/pkg/objstore/swift/swift.go index ca74ab0f8f..37bff9e6aa 100644 --- a/pkg/objstore/swift/swift.go +++ b/pkg/objstore/swift/swift.go @@ -125,6 +125,16 @@ func (c *Container) GetRange(ctx context.Context, name string, off, length int64 return response.Body, response.Err } +// ObjectSize returns the size of the specified object. +func (c *Container) ObjectSize(ctx context.Context, name string) (uint64, error) { + response := objects.Get(c.client, c.name, name, nil) + headers, err := response.Extract() + if err != nil { + return 0, err + } + return uint64(headers.ContentLength), nil +} + // Exists checks if the given object exists. func (c *Container) Exists(ctx context.Context, name string) (bool, error) { err := objects.Get(c.client, c.name, name, nil).Err From 847ae8545ec5513fbb61770691cafbd286a535ed Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Mon, 2 Dec 2019 19:30:29 +0800 Subject: [PATCH 076/257] delete broken link (#1821) Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/quick-tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quick-tutorial.md b/docs/quick-tutorial.md index d37f71ba9b..42d51bcdeb 100644 --- a/docs/quick-tutorial.md +++ b/docs/quick-tutorial.md @@ -130,7 +130,7 @@ Go to the configured HTTP address that should now show a UI similar to that of P #### Deduplicating Data from Prometheus HA pairs -The Query component is also capable of deduplicating data collected from Prometheus HA pairs. This requires configuring Prometheus's `global.external_labels` configuration block (as mentioned in the [External Labels section](getting-started.md#external-labels)) to identify the role of a given Prometheus instance. +The Query component is also capable of deduplicating data collected from Prometheus HA pairs. This requires configuring Prometheus's `global.external_labels` configuration block to identify the role of a given Prometheus instance. A typical choice is simply the label name "replica" while letting the value be whatever you wish. For example, you might set up the following in Prometheus's configuration file: From 6aafc7b658eccbb3816e1b395f314aab1bfe1204 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 2 Dec 2019 12:32:58 +0000 Subject: [PATCH 077/257] Updated release process; fixed links. (#1817) Fix: perl script was only changing first item in line. `/g` was missing. Signed-off-by: Bartek Plotka --- docs/integrations.md | 11 +++++++++-- docs/release-process.md | 16 ++++++++++------ scripts/websitepreprocess.sh | 6 +++--- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/docs/integrations.md b/docs/integrations.md index befbb9fe4e..47a1476d69 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -9,12 +9,19 @@ slug: /integrations.md ## StoreAPI -[StoreAPI](https://github.com/thanos-io/thanos/blob/master/pkg/store/storepb/rpc.proto) is a common proto interface for gRPC component that can connect to [Querier](components/query.md) in order to fetch the metric series. Natively Thanos implements [Sidecar](components/sidecar.md) (local Prometheus data), [Ruler](components/rule.md) and [Store gateway](components/store.md). This solves fetching series from Prometheus or Prometheus TSDB format, however same interface can be used to fetch metrics from other storages. +[StoreAPI](https://github.com/thanos-io/thanos/blob/master/pkg/store/storepb/rpc.proto) is a common proto interface for gRPC component +that can connect to [Querier](components/query.md) in order to fetch the metric series. +Natively Thanos implements [Sidecar](components/sidecar.md) (local Prometheus data), +[Ruler](components/rule.md) and [Store gateway](components/store.md). +This solves fetching series from Prometheus or Prometheus TSDB format, however same interface can be used to fetch +metrics from other storages. Below you can find some public integrations with other systems through StoreAPI: ### OpenTSDB -[Geras](https://github.com/G-Research/geras) is an OpenTSDB integration service which can connect your OpenTSDB cluster to Thanos. Geras exposes the Thanos Storage API, thus other Thanos components can query OpenTSDB via Geras, providing a unified query interface over OpenTSDB and Prometheus. +[Geras](https://github.com/G-Research/geras) is an OpenTSDB integration service which can connect your OpenTSDB cluster to Thanos. +Geras exposes the Thanos Storage API, thus other Thanos components can query OpenTSDB via Geras, providing a unified +query interface over OpenTSDB and Prometheus. Although OpenTSDB is able to aggregte the data, it's not supported by Geras at the moment. diff --git a/docs/release-process.md b/docs/release-process.md index 49c319c651..8d56134ea4 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -31,10 +31,10 @@ Release shepherd responsibilities: | Release | Time of first RC | Shepherd (Github handle) | |-----------|--------------------------|--------------------------| | v0.10.0 | (planned) 8.01.2019 | TBD | -| v0.9.0 | (planned) 26.11.2019 | `@bwplotka` | -| v0.8.0 | (planned) 9.10.2019 | `@bwplotka` | -| v0.7.0 | (planned) 28.08.2019 | `@domgreen` | -| v0.6.0 | (planned) 12.07.2019 | `@GiedriusS` | +| v0.9.0 | 26.11.2019 | `@bwplotka` | +| v0.8.0 | 9.10.2019 | `@bwplotka` | +| v0.7.0 | 28.08.2019 | `@domgreen` | +| v0.6.0 | 12.07.2019 | `@GiedriusS` | | v0.5.0 | 31.06.2019 | `@bwplotka` | # For maintainers: Cutting individual release @@ -43,11 +43,13 @@ Process of releasing a *minor* Thanos version: 1. Release `v..0-rc.0` 1. If after 3 work days there is no major bug, release `v..0` 1. If within 3 work days there is major bug, let's triage it to fix it and then release `v..0-rc.++` Go to step 2. -1. Do patch release if needed for any bugs afterwards. Use same `release-xxx` branch. +1. Do patch release if needed for any bugs afterwards. Use same `release-xxx` branch and migrate fixes to master. ## How to release a version -1. Add PR on branch `release-.` that will start minor release branch and prepare changes to cut release. +Release is happening on separate `release-.` branch. + +1. Prepare PR to branch `release-.` that will start minor release branch and prepare changes to cut release. For release candidate just reuse same branch and rebase it on every candidate until the actual release happens. @@ -88,6 +90,8 @@ Process of releasing a *minor* Thanos version: 1. Once tarballs are published on release page, you can click `Publish` and release is complete. 1. Announce `#thanos` slack channel. + + 1. Pull commits from release branch to master branch for non `rc` releases. ## Pre-releases (release candidates) diff --git a/scripts/websitepreprocess.sh b/scripts/websitepreprocess.sh index 5f4d1d90c8..f8d5b45273 100755 --- a/scripts/websitepreprocess.sh +++ b/scripts/websitepreprocess.sh @@ -78,11 +78,11 @@ tail -n +2 MAINTAINERS.md >> ${OUTPUT_CONTENT_DIR}/MAINTAINERS.md ALL_DOC_CONTENT_FILES=$(echo "${OUTPUT_CONTENT_DIR}/**/*.md ${OUTPUT_CONTENT_DIR}/*.md") # All the absolute links are replaced with a direct link to the file on github, including the current commit SHA. -perl -pi -e 's/]\(\//]\(https:\/\/github.com\/thanos-io\/thanos\/tree\/'${COMMIT_SHA}'\//' ${ALL_DOC_CONTENT_FILES} +perl -pi -e 's/]\(\//]\(https:\/\/github.com\/thanos-io\/thanos\/tree\/'${COMMIT_SHA}'\//g' ${ALL_DOC_CONTENT_FILES} # All the relative links needs to have ../ This is because Hugo is missing: https://github.com/gohugoio/hugo/pull/3934 -perl -pi -e 's/]\((?!http)/]\(..\//' ${ALL_DOC_CONTENT_FILES} +perl -pi -e 's/]\((?!http)/]\(..\//g' ${ALL_DOC_CONTENT_FILES} # All the relative links in src= needs to have ../ as well. -perl -pi -e 's/src=\"(?!http)/src=\"..\//' ${ALL_DOC_CONTENT_FILES} +perl -pi -e 's/src=\"(?!http)/src=\"..\//g' ${ALL_DOC_CONTENT_FILES} From 5b7e33e81121a02485fd872ab2ee9436c4275050 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 2 Dec 2019 12:53:03 +0000 Subject: [PATCH 078/257] Enabled stale bot. (#1795) * Enabled stale bot. Signed-off-by: Bartek Plotka * Fixed master. Signed-off-by: Bartek Plotka --- .github/stale.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/stale.yaml diff --git a/.github/stale.yaml b/.github/stale.yaml new file mode 100644 index 0000000000..3d454553ea --- /dev/null +++ b/.github/stale.yaml @@ -0,0 +1,16 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 90 +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 14 +# Issues with these labels will never be considered stale +exemptLabels: + - pinned +# Label to use when marking an issue as stale +staleLabel: stale +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. Use `pinned` label to avoid closing. +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: false \ No newline at end of file From 8de02fbb4da49a27af539c4888077414409dcaca Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Tue, 3 Dec 2019 21:43:18 +0800 Subject: [PATCH 079/257] add partial response strategy values (#1827) Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/components/query.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/components/query.md b/docs/components/query.md index 2f0a29771f..e566b4cb0d 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -140,9 +140,11 @@ you might lucky enough that you actually get the correct data as the broken Stor If partial response happen QueryAPI returns human readable warnings explained [here](query.md#custom-response-fields). -NOTE: Having warning does not necessary means partial response (e.g no store matched query warning). +Now support two strategy: +* "warn"(defalut) +* "abort" -See [this](query.md#partial-response) on how to control this behaviour. +NOTE: Having warning does not necessary means partial response (e.g no store matched query warning). Querier also allows to configure different timeouts: From 01a700e8c83b9de949193f8d95ba2dfe01307ad5 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 3 Dec 2019 17:50:46 +0000 Subject: [PATCH 080/257] Pulling release-0.9 to master. (#1831) * Cut release 0.9.0 (#1829) Signed-off-by: Bartek Plotka * Moved to -dev. Signed-off-by: Bartek Plotka --- CHANGELOG.md | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05359a22f5..3af82e0be0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased -## [v0.9.0-rc.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0-rc.0) - 2019.11.26 +## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 ### Added diff --git a/VERSION b/VERSION index 1091a9206e..c70836ca5c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.0-rc.0 +0.9.0-dev From 2190d58d379eb774f7ccc5142eabe9d929bb9e0d Mon Sep 17 00:00:00 2001 From: Alban Hurtaud Date: Tue, 3 Dec 2019 18:54:10 +0100 Subject: [PATCH 081/257] Add Amadeus in the adopters logo list (#1832) Signed-off-by: alban.hurtaud --- website/data/adopters.yml | 3 +++ website/static/logos/amadeus.png | Bin 0 -> 20086 bytes 2 files changed, 3 insertions(+) create mode 100644 website/static/logos/amadeus.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index 9e8a71b705..77d4d90e72 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -60,3 +60,6 @@ adopters: - name: FREE NOW url: https://free-now.com logo: free-now.png +- name: Amadeus + url: https://amadeus.com + logo: amadeus.png \ No newline at end of file diff --git a/website/static/logos/amadeus.png b/website/static/logos/amadeus.png new file mode 100644 index 0000000000000000000000000000000000000000..bc6e21f6362406de5474397922375ee1ce62ec59 GIT binary patch literal 20086 zcmZ6ybyU>r_XbKMjdU{t(j{HO(A`6~h;%m+BOoA>0wU7g-Q6wS-O}BCKb&*EzkApH z$K@K=a^BhbJkQ?yn~IV&1}X_E3=9m0tc-*j3=BL7_`CHhB;e0TBmU_#OUftv>Y5&?ra@Jc1kEXJgNc)5Hd{d3B*^FiQ@huj(ewPFAD{Vku` z$9>O~&sWFGoKSwGz;Q$-xoA2cB+h}T4U>yC3-I(QqhD0Hnz0Q+Go_>mH4La!OP+q7 zD1g?hG11oz)H&1!8ji9xhEo0xp**tw;_fEFJz{z6h98hS<{Bmg!W-( zUn5T_!)70rEw==R2=171Hru6hErql#=L&hb@?Hxy7J3aUF_pI4VH%utiFStjq?*zz zWj|9$ZrKQBFA#Z&L>OWn9SYKyeS^(B>vvQ8ZJ*0!pM4Dx)@cNL)4ndLTNto*;J_`= z#HTe$#8Ti%P$!mb>%?AThneH)qog-UKJ0LiYc=u7t`D3KY`z#5TbXPBNR0H;xZ;6I zuNOx6!{P(|7{YOx#aGKE{J*Sk+wn8sXZR$cq?{W6>IU8LZT>+E>C7jm5Z@Jvc(UPp z49j|s%y?EOs0?7hN^fMbvPNj`t7-Ul=0*M3PJ4Ghq)>)*S0oC0hHjG{cOR!NSsZF$ zDh?1#;(u@2G>Ogap0uSD^RYsS@j)K_98`fhUie2|6BK1gJZPT$$aA43-6-a{tiV?M zQA@?}IKw;s*&cT880J87&X>wY8oNTD3$*b$uuT!#)RI257!`BrF}ByFT70^OSlk37 z#Pa@bTTxG%r-IJBX8!fOF_BMqC-r;V;^51xM!1VAE_|ZTwM!zOBc~kA;44_E2y<1* z(@&8V5Q1>?hgDibGV^P~a~jlU9|g%xvF8Zk7DAIvBZrcG-l#Q(k`irDzOpN_)#soq z5qzQuZ+2t#&Pe;HF!7E8sdwx-B3rckG?qqKE`*xzgsQM3O*ou{{sy|-BKqv9MD%cs zA^$+ByR~BH-QG9Yc57L1;F?4CS&DrlCFpm(Yiv#4K+VJnVrINbee-X|5k#0p6*I!G z9&ag1_zdEjZA39OVY^UbQ+m~?$fPm0-(9rC`f^8;sMu+cRhZC{XWvc$AGT2%gB@*p+S>Xkyt{q!IWQ(elcK&Q+8~ zX3f;=Iog{I8pzo3`P8T1RSBCu7=bEN3hxWy>n1J-vXwWRM3l;x7R+&LRC1f;3d^Hn z%o#J~K1&XV9dnmcPlU`UhHoZQ4H3zFS5{U=SL?oxXXS77i1$Jh4r6$K@}FUl--g+U z<4C?b+uD*3!YNu0^^_BBSma!PTW z-QO?Ft<5`2uYR*z7tmjm{7NB7KD>>=~kM{ zA`$wRzU_sYG1}@W>S%1@eNmA;EJ+;*n;F*5JF1!(+`l+)GCrxJ|KsP`xvn#mFt5%C zZhD%h;#qX2aD}AONqRBrh;ZtWa(=n7mpknKq`Kx4AZwhvjwY$=x}2Tm-w=#qNMWS= zL}(yf)K$ywg^3?ZM?WOBbM__|v&Whd)n%?WU>woa5c9@!(}RMWr{euDD|dy2vgdER zsd)-3$xLPqygf#Sf<5fFTrh5ouiQj~S9eSWKGis`BH^Oe@s?LTthn`>;;Y%<5fA0U zgv7wEDctSQd^(+wL^j<`TOteLM?Nb_}Ouzqp{^F_;QV`*f`pSH#tx1?rBnrLP zbBftqEe7z3xVpkh3W>QIc9%C$!V_$9(ffNJJAub z@A*yc76x0Obsl01fxtAx3~Q3@hvA3PiB=Q#H)^d#-kso;{=7_vl2CH9SLmbh+bZq+ z47j9oNZE(d{^iyrR&ahbfr$hFC}A0ppzWvgW+dks_84~i+kqDf8eLJ#fhC*SJ+zpWS;1l(8ug!HGwhet4@^K z9j&(uTpBy*Z)J)nXc)+)pcWbucZ-SRazG3js)NgNuJ>UghhW+Oc23Y1JSan68x!fqR~U( z-%e3Bkm}9V8X<{2h08}BJ*|b zeij^eMQe7%k#6G#QAkC>toiPiXXAzg<0PaQH_RaUP`SYX_6J7^OnJf^CNSLVPQ5;R zxHd}6ub0}BH6`IQ2*PhGkjR8?#G)}`c{VP;#wDwKC^7$22QuWmHf=POB^;QbR84tW zluL~@#8O{NG|e7q7fY7J;6@i^@}UZS4xNE4+GLNo&;v2C!hT}!*1INw5E_gV1BW&n z4mya)X48lLK_X^coWOh|Rii}iiYPNs2iNr}Czm3$;c(dF?02gf zC8-$n;HLCh9Lyp)Bkeh~w-#^-eAKF(rR_)B<6lVH1@l(@Cgf6`BmteJQ5x#xeahB{abqWf3Ctn>>a@u? zWiEWJe&HbEEk59-e}W{P3cPug**0Wu$?TK2RIZH?Zhe&FI5gAfmog5gX2gC*<=~rh zSLTGwxO1MLX5Jk`WrZ-KLe1BYxk9_Hkjr*b%%&^TQ3LpT8hcet)V0UZc^|;Vc678Wz#X15tZ?gd=U8ptsOo(Z^We@zS>HTX= z@5b7d?7CnPe>S`YK8Oux#T7gfbI_8{V{#$9VW_Cy%FX{t(_$nHyVXe zw8Ond)(epN-8;>ecpXvv4jnobtn-P^QXCmV>MxkKM|a-D_-yY@+B)p(e~<*5{QLJvYoKs6ehn@ zA!$~&-0a&GgbW*{JNXtQzoQ}cZ7CO6J*eansM+{R9;?}TfZ5#-LXCMh0?ADx+(gB8 zIUtNzD&eiHjRa%5a)2<27qA*Lya%RPrb0DKxFq1Zig z5Fx&fid{Vlvppse?&oHIo_N^xzn`oDyN=QG$zeF9sSEX1 zXuq$*$KhL6MYk3U;4!0G!HDG(q|B9VGG8-yGB_P>*G06$h)gCove8*mu~pY^4y!Y6 zz3;GFR1U0R4(xzhETH7IZe9%&oyf@kao^ILLb?Zca&3|m594Cq*lE@p)fAW4*dDjy zePe<7@t!~w+Y9{YT-2i1oh9l#QFv<|)3;2sSFMB;l&j|>V`22F-8bdcy`POcnL)WO zRHcPWy~j4qdr9Vai_e6KL%$cBRft#W%vDG%6f4qlGPpp*mi ze1m5jC)Xd&bhZ5#_n~5tGgB&&5}?RQi+frG=fq_YL=YRa-Oze|u>w zg+BF9W<-2#MdukiapA1itx!yR$F#uUC;2>*IU=o+z$gY)y=X+aaLkVxW|gBI$|Q*& z`n+<)=YonokQvSe^T3Jm>KUH)InAJ~w#rtg(ZbcxanfLhE-~rnV%~B{J%3q zl0L9yI{wLf^xJ!}q9l-8WPi?Atu=2m1sL}4IPN@YT0Czm5Bd{|6J~eob|>xWT+O zx&oOD28l0}!V5h~fqa|+G`qbsAZ$Nl854WLrrsgwG{{|WpjvPc!DpE`c{(FQzGCW5 zhC`?LF3aVcY6HSS1T{XywwDQ!p}9%)@(SK_8wnN{Jq+uY3=b_T*2*6OWozZZRnI~p zv+RX|sLr*%ZqJMU`bnN7vv~T~V)`+!v=%Lck4tv&L3K&<&4>avSgXWx1s}Fa;aV}0 zjBJ{phb4E1QkqO_UYIlaQ)8wMR7mO8MqZ#0Hr=H?R4rgbRkmlP?94${3wl9Ah#10# z*N6tawf%g%!NMc&5!U?3x!2j3#275jcCFPuLX%?q<*z8RKsJGFYw`3Mai3Te0H*dC9Uc2==!b}9>lK%YJYH5!C4>)yx zdY3K#`%)AO5-gm7=^MJ_m?g%za5{cjlzFvE9C323$(6cN<&sjIo2Q(ot3mIu@4j=W zOh3rNe-S;tDn6h-IG@_m=L+itV`O$PNW=APyfw+pkN^FR;1J4T3lli$8xz>;hc#Rl z4bl7lPSJf5m|K8NnN}xi+E;PjBpa;r(K1~Y`ITXK^nq>xhr=gRiCw(wV-^sFHt&!5W+@u zkBQM~KV_C$va9?}_Gxb3{B^c$Z@B)dmP)H1B6}TMoa?A18aBUHN()5M{@EYh^svKY zF+W%L+IVbF$bNAC7k=dSQnIRAfIuZV879*hx1Zw+P}l04hTqPvxj7DK?l;LKs|5SO ze9V>O_Y{DWsE6WT>IFSgPe?jwPI#(7&ODhUWYkvs!`IqK8y+uu0USxb2wkkDbrZyJ zU~cReg%7;t+VLD^^lM&2#Wn|UP|9Ws%vkL(!PnWPc|(WIzrn0?S)DC777DHwPa$jh zVMgNwptFc=LsP93b%KTL!8c#}@?J}x&~n51P()=C9vA?q z67yR%BF(go4ve9$#`W8PHB+~nqe!=&h-H;SzmX$8HBsRp+aw|L;EAKYtWainvhLI> z<#yZyv`drL;$SEyKWqi1J^LN*@ER?Xx9Y<4u|)VYf?%pYz!nk!0!YVC2NoOSE~u@1 z4>Q$lq`uUs6l^Q#(3Rv~m=1R)$p}*3y%3eV965F9R!9~e&#tOC0_bdSa+zY>9&rI= z;_&1ULz1%Gw0#f!i&lR%%l5r9xjntw(3fDBJ8Za8g+Tc0{fl?l;u=tyCXKO1{(w@Q zwLytjT$8cAMJH4Lm!C=~QIdt{-YnL7F3IsTv^VcaJDthharxDQvS6ZB#7~qNY36GN zfUc#C12W&ONJUvL=z|-+Q3LFwv@Jko8aic-*kgFtSLgpwgAfNF8Rx4m%C4G_SEBvN zolG3j$fA3V%jN;REm}xF>%L*~Y};^<+o3`LauWFIZflc`&r3$nVwhXSCs~`?0#IX4 zM?@KACjg(mlQa>%mm;s(x?mzvk07~Ws|*JCUQ_$5_i#cDEJcQgprN{>KkI!07;dwX z25eUjpqR2<)m^MS&YQ1s!s=fU`mSZ57J7Yu$8j04BGo-duO@#W*@swj0I);|GB502GG>UaM58{8ok_<4$;Xr`p9C@#b*!G6=H-q30uHE>cWR(R!l__Q^Es5#!|CGeM?@x5A{@ANdMFfh-vAvSOK_R%%3W&&Hxb; z0tma6N_`Y^am0kl0zPOs{iZ@1>wL4?SHiQ{Rf&6Q<0~Yx(DiNys$5WspZ{~7sOzRZM0$AUpKtC6E%rPlDQf&L(_UOjFsXFufo{97y{)@M zIu4h`MU-gia8-x75P=73-!UxM=>)tdqEbfL-RWD5*6EWzI|DPjB;bQifH#YWg)B7- zm@)wA)u08G-6+@=_yuU5z8JwD^|!-c$;C|bIF4Er=9<+1zhAgHG^F}xF?}L9PJPOi z6R;y25kD#4b1>$D5-!LTF8Z}(XWe)w^)AWRJzM$yza(SAo)q4S*@^@EywT;a$-wyz9AC)i%DSG@{PY*Us6PoryN?PQe+~lh)Zx9>2nPLp8 zf;rWkChx`Y)(f`HMZvar==MzdZVs8xvi6R9s>kAMl`c21g(oJ@eUpi;^$yCEKgO?- zZSdx18)vOPVkw`A@H8lcHzw*K9dqFYYs79-=C^o`;tSS%bpKx&QH1UgnO+1fnkmCH zHl+)wD+s`TX_>V=^xSVirGMg7`!7Eif+76o?<+ zFJ{t2j7e?Jz&oPPaK3twjKGMDQj zYBeHNXg6}#2_yi($KLmI!>W;lYLdX16h5f>=r_4j8rD+lrf0Y81OF;60afXAXW{m) z&9C|fM;R6jf9^`}+kQg&ZLjSZGS7_G;~!Q@634?ZyrQ0@wXfZzV74O&3G&Sx&yjxy za@Dy)1G=*qwh0rTc1 z+uQTyX~(hzY!5vuvYV0@?{A<+{pr_)rQ+S8XZ9zD_`PrNR{88kU2XoHIq2)I|G#2} z3&I)Lj;BvmuAMJk>_R49wjm&3T`BX>tekq90E!Y|1d9f7KU{+ijQc|-5segqLtr#x zSlpp0>(CvI^y^qPY+yjh;2F*3(R}cFGUN%1JA*f`qItzv6LY5m(3w6e@x*KWr{u8S zPyQ^$DLY6^u85-IOzWNnx5XcHEK5ON~#u4=846pL)1+#gzM~C0D&kj{W>3=Jp2CwLHf$|m@#%2!jVSB_7p4rc8KiB zM~R|?nr*68#wt)2dnQRzf4EEihTHTDTC zd5!B2i%+d)W1&2PrVcO2K?CX{jA%j$cXH}>bBtEcMEQ8{?xiID20`XDLppe?x`sQy#`?F1ROfix zAo9~OS}|Xe@T1C>0NR_b^KTB_X#^34b{B$7W$)es+VTc`swd%evYh`FzJ@U7s9}_d z5Gg?nM$mBk_%Z`+#~8#Z!sON4HV3%37kEhQ3fuV=nt*(>?IZ)@u-asAQP0Kt@^|;! zZ_j~+kmXSZa7Ek;NUmW^tX~)=`Uex>gK+HWo%oS`TMeF&#VDx38%Mv`Z^whOowcHA zz)X+UgdnqXO2I&4K!P}}L|oNz@uX8XAuU-bUE-64dN5f>t_Gz)NE|LJ?d70e<7w2f z6;Mgac~% zI_3-`dJC6zRrBy}$zA(Iv-^|!r&7o+GGgOfiYs%j>hB0+eleiC`~eRHQ)is1r(lt- z?SXl5&+*xu@kj*Nk#IH?i)50FL==re)UUGw0lJAg=yQ^jb9f8nG30#doGyqmE2H2w zgT2D4;(8qV5}8SeHiv6JP_kd@M!zepn)h60u;-bsQLgv(9IIDA zUc17lc_8MiYKTa0Cq5@2itrNoggF1oWk#yct)a6Qdw7})Z+=W68Nbed^Ik2k^-)-) zdf{hGwBVO_ZF;gEWY+G?N?j<)=}h3epuJ<%f>vjsVLv>xY&};u9Jm9W=pWA$100#^ zUIr$XDy!{Cn?p2h7CCHX0dD;-%M?B-vNfA6Y=xijHP-8s2Aj`GAKymcfZ8p7F?wXn zCA#>Oj$bBE=Z>gkMY!Zw#Q1GBNtm3Wyv)4wADZ=JU=N#A%_MWNHM^`C2k!5hBN)i& z9V~^&UIB3Lcf`^NyChsB1Pe(Y?^d0h^r973X8Vpg>NtDvVJc*8P3bxMcF+#$;aBW$Y)A%*8{v z#K^YBJOtUF7jo@tOs||k`N^G2srO$y_TYo+!+`?buL5-WdR!VpqVx)fjHPmPBu@}0 z2D~qhef#IndW3@53}{gmDiaQ@tel9=QKEJPW9_92s{j`D1_u<6v8t7VGQQ7M ztdPcKKv;UJtYlVdWlw*{gUqLtejG_Jl6!H?`9f|Xw~NQu250+x?JJ&rI9>F9_ju{7 zsXhPy=iq_J^=~A;*xO8yLnsL5$Onwyyg93ZX~|2Ue^wJ%kr##bH71oWSg>CoDg=;* z@+m-Vw0wS%qBb#6V0u>?wYJZ?LsGdSX}#aIR5h*8F#(+aMTxt=Ppo=Gy;|~^_0wzh z4H#Q-!dy|?t4-~GavSrkLG@!^I9}Ec|1VjYz)qw9QHTL}>yKf%*N_1&x>j zLfxBpV}8z&ORD#2ey|l5t|{&-^JN2Ho5T@&Utz1wnQi>mgCu6}H-0NQ4tjdZp{e^& z1^Y>Cw*)I>?Cv*M($7x}W4;-Z(3&y)w&s_G2f*3y-){#X^eN|+WN}F{lkf3jFnv@W zw$ZTV#l1TAy2jhee>4*ubPzM&L5*qwg2)aSjhMd>(9J0D$H@?}8)A6<*u%ztHIWl4 z4%O%ZI~^aiDAK6=>7GrpYGJ~qhKq1@guc3gd0E7ua;fSiNNsR7Qf<&cRE6YZR>gwC zMeM~}q#g}nZlToUd6_tmK!H@EoF*~{tGHUmJJ;b=Rt;aQQLEoEye!GJn$=4#(cg!n z6Gj}NIE|+LQc0u44RBfZ!0c+_U)Hesyt6pk{a{&h%qqW8#X`IOHhj(V`QZ5Y4i5y3 zA!uQ)a4*v_Fp{XiRTz8-Xj8$DYhRvXwq;cgbSssxa>%V zssqGUqiWF@TitdB??4{I{EZhyx85tME8ZEL0t+=K6JC{W$|Djd^qZ`^SQ#osL z?>JW@82D7pt$P?+aX)d_G$p`RDvvbQ^y`7cqxc{naob*8f!ZH#SX5Pn|2=HzW;HbP zy)kRXmzy|j53+xjhR+ASkFAjnc$(gf{c~XjBFlYv`1SZ7+Q?gOXcUCBwIt-7G8s`% z0}|XWAR)i7_VLb_Z6RC*3TBp>Z($XJDUh_5FY{Czu^vN*w_-wr7FZG#KoWQv`^~K2P(TJM1Ww))n{r!N zIP10XaG0Ks-N*}F9wPQujPql_)V(TSXBi7^CZ}Z9P2L99+hnM=*)@d64Y&#lCvv4+ zP`ESV3u_WlAtD6ok2ijrcUcV+N^HlYBi1yi%aT1aW4Q|oo22hYTRvlL@{<3-w9_$q zZ*Y5LCxE>3zf_vu&o8=nnIrUPkqLYp{Qp1}K(YPvOQpI1yL*}q%~+JA&uI0c6PoXL zqt)avZfSE`bp@a5iVKSNa~K|}q5nnr+Cj4M8`Ht!8r?Ob%%&yg16sl1v@$cV+R}bK zb;Q@|UT3alHDY-FXSs}dzf|v1Q`kC~^T4hjp7y!tv z$p-Ge99XEbwCOSD;4jvB`>zX;MxJzNxBAk^hzxfoi@h6_#UCZ0v6|D#_CzccE+`Fmq=xqU+V;t@5XUL%GGHkbK`N!9Gc z!qKbm9|aF@S6#vrAsFNGH0zo<*$r3c|f<--zk(FGz z$9Ecg)Z>pSwTu-|&K8IUnNxvM4F5w}bV>%#7rokiZv=*4c?u9MX8G&Skd)-MaNQIB+df-h(@g}R1Mtr=gF_2HNN>HnD$%4nJUF@A z^4+rT^7@~i0=!(fNo$&7V18ANWJjxDk=%vdC0T9x4fqO+hiA;Vb960=q{v+lW@JUp zpjt45zS45^a8NzY30kHeg=A7)f=0~@?Q`U7hxbcw*C zmRAbrLnZ5dEDseh{47(Lk5%}elnt!oV7x`M$|XUHC%uO3&U!FtG*Hkky#c!eY755j3ggr4d&QkcPfr{esB2% z#df>&_S0+5+Y(+ppFsFOYWRb&^Ye3E;e(I+8ViSLM)YS|IH80)Wr|zdZx}@)7A;5q z4U53LFV^{#b&Bzi5s4(+_&;rJj{y*np$`4kH*BAGS~_l`2Nde^U;pY0!;(F|8O}4U zPkau)T!LZ7G#JX@O}(guu63qrG@=LLvZ^?|8cf>c;h)XC&_BWKzR=^}16Ug5P7JvL z(do?vm;$kHf4(!7yFOf3lbk6r-A!ub&>r|J;`Ykx%2u=am!Vky{TrU|BQzh*ukve( z;a-FsO|&TE3qukFAITR5C7#}vIT7wS$%X^&)O(0AEWyit>6S+pLdTDr<^NanbJ0CzW$3rf$(V}A@nMC>h zC-q3CsePw7snZ?OHIDzq#t!J?YSl~2qZ)QJO2;(7iW&dh=<@Yfz{GLz3KM3O#KPF6 zm=;T;>HjEf1-$aRsF#D4`9{7H!M4c&YO%xiPS=JmFDi8A_5bvcz&i{g!iv3}pg%X7 z+Uz}b#&On%dIuo=a4)ElpK-%ASinf|UD3l&93rm#$ezsVU(Wx{>bdiu&mP~IpK!*# zlPwbJ3PUn&O>dvA)mX@Os!SOel9+PScqyIzy0GK*Jhrvd7=#Fh@vo45i~kecESk#0 zdPvV*XO9|Ke0W|g$9@+YEKN{dyE4fCr?Ml}c%38Py9KH3NYhq`6p8dl5&@`v_StFp zNxs@Z;_A>N{P`*jqc7L)tIc}{!vgjq(Eu^7F1^{n_5=K5G^rN38P~{xIvTOERJsmd z`K#qzfm5NWZ&kG*iV&L}^ncQq5u7)iqUd@>Dd6)9oz*Un)lWY+vg8HVS>S`b*!OD*1(>KYbR?;a5B+%ysZ~eT*I&7`(PJBNVTP<#W7GQ!$w>!^aI5JAec=Y$S#G@|o z?ZV8y_D&K?tdZ>B{zUd!QU4P7_Y5t$c|r`Oxl7S#qdJCzZsawI=p&*FN2$%y!L}Oz zDY;RrPgp2NrY|b;oWw4NVf^FrOBqpZM*FC>`DWzwcc{~UmJ{3P)wA%vGBCKxC-=k} zD99)z4p4`X^1e3bnM9~MEG!8nBLqFGRSQD2AcuVt7$c}w9)I&fEX#m()l&!K(eLo` z{$nvhWQJO=)wS}3nq{|kfzWDH$I*Ap3jTf#gxig~+Cy+&O(DrY@+xXn#cg>%bEH|- zhCxA}pie9-;qJv4`3nF|R(@hIGR*eoSNt~*(p`$et?$6SH`-$Up5MLUDk8|}De2MJ zm1UR{u4}9s3;G>lD`b1WaLTsl@&s-BvTF;Pwa;!7oBOH0o=AKBGCh4ZLblG~BVrib z(7*L0hotbRdy?eN+qugzW}17VtF$;GxFS5~M67xM+3=4BE%gHPo)Y0_1ng7J9`9q1 zYj^(ly6qugpQ!hGO*BBK(*hZQp{y;&2f=M~LjpGHo=b0Mpat?SWrOgd!Nl2bg}|+Z1jEx6{%6W*6zn;vBfZ9yyVeiMc6kS> za%O(jN|Z}4n$w&1dwiUx>%EG|zR11{bf!kY`_2r8D(+4x$VL+;jp9(1;6Q;eV5(&Z zt5)HVJreM#8FS%ZtqFKmlScHhQx8kC;4FE2*EP-*er-X;_RDIz&29ohW|3omQk_+! z0WdX*cZ6;g90E!(@rxGngax2rx#3w#*E=xY!bS|7DWp0+|0O@y3&VVf(XaAY?=Aja z)^(Yi`2qk|c|e!9z#^2h<8hZRgb{6X-jIjWt>uA;mY`xEcQmN;uQL@K7NiWaq1C~D z@;^&ci5qfB+qZl-b^;EgwX+s^zsuGy~ zq#_*97nzpYSIg-YzI95xV#c;qWOb0-$bS-op)fS5gJ|Z8OJPb^iV3isGqIoN;V*9! zn`slhh_}@!*tl;-Rn#daO!VIJGl99tucYzsObbTkaurl5CRHd9H0BlX2XfiB)PWg} zLE|3!Ra6&blpd_R9`n#y+=83L`tk%D9FV3he1yLN3D&)}>;3M_fUUo7prITQ$!^@t zsHU>9UA^fAcNqvn05ZMJQBvWi_Qj-V40uEI7S#m-$pgLx`xBE@mT%fiQ~{Hkf=Nia zmikv=jqITx#)k^O1nE@w}Q#svbl(o ztsU}!iK@hdg1F);3wezOu=)F^p8%HXSd6+uBtPQs?uGbebv{2^>1;%CE-)a}cTpUB z5A$vL_{Yvv#i#)?um@x{e0vOBXow{y4(;gT&ZD#-HMq(B=-DrXpO{?GkRL{kQn`?7 zC;FA!lOk>;0@ifyfk1@wdfV<*mDUBR;T$G9($#*iGl3=Y=-AfOoX zLk(9=*msuNyaLvrF}t5h|Mx;9SSsYdo#8q0dA=0I3xwId9CDFzOTPa=xvzJ@+>j^Q z-HJ$h9-aj196KW(=UP@u#wbB4FPIXOzLN3s*FU;YZ0&e9FeLqYV10`^Yp~Z}gOE#q zz_W;@#}UWyXpHFOcGfUDX+!(end4HEimjC@*@$i#9ZD{beBE8hf95kC<|`~Io9)Q~ zwN0&oou?bjv%~W9bHMn0YV5N4ZdoxZP~9Rn5M{Q9p=S=$=fw`@97)S6%L=PW`LH}qv3vlVf36s_AQAW^VV!>fVp|_QX@Je;`eCF!{ z+iRqZ*#%mua9|%P=#0hLHQ7xb{zha1#y7{RF_j9t{~BNPd&FVlTUlBqtD=${V0_C6 zj644F=|cO&t9uT@RTC}x+Y-jt4dl>la|^KofmrfSkb~S)lGd|)102Sa)?7HhILw8m*L_E_lQ*q7ojl*> zDMxP}2@4K&tK(R@TYs*n=5_L*w6=CzGYXe>UZnx2A-%jN3p17uCwk<6lShd$CI3%u zr_AAa)G74VEjKI#wwc%U2mH%H8%0vJlvdV?Bn$cmMu$d43tIMJsg#LP+po{_m*Y5* za%W!$ym$gQLSaGNEZxKVPFqHu&V^UZ1yhy((W=O>=hM6QuM=yP<0D;&UbgfDRlbD9 z@CPXL18{$EapeCt7kAc_bmrtK z*k*8KdjqSMJ6>LHFbxruWZ{V9^1r9o0ZXLzU`6gdBshR@~eeJgl zwB&+^$S@PjUY7kdgd1O~g_vmxuEn3r9E~UX))Xj>ccpK=(V(4FDt5?bJi+ORQ_!UQ zYP`zz;2zsD$_JWZcG41_o92PT7JMS&_DO~JZ6vt`j**;;$hDPJSObhURe zL6|I;G?XRtcWh)Xwk*glwFWf`+;xI4n)vfJ?l(dQN(6hgi1EjX@;;x4@ zl8)bRg#Vb2p{iFOmUt9qi2OzO%^7SFno90R++9OuQ6^CffzCJg5hI<<^=a21Re~^A z1BV^*UPV_Kcy`0DSgQrKH(*z3n^mdC07i?GneE+2(M0csaME zSa^+3RZ0HP?NPsxoK()dV~U)P8b%viXM6C`h3$5ux-Pz8fVyCba|{AXT(4k0Mm^F; zvzQK%iQicsj=+|1P40d=*STgdQjJaGk^I-N_BTk2AoIy9CgkLeSI&7{e~ zM+Zl7*V_#N8+jLz=$Kz2?LoiW)C{iV3o}wViHgQQ2%!KY{sCT#@@8PM!i^RAsz`|v zr-1r<%vZ!2J8hvtom#dzo!>nzyIFnP>}|vi3ym+0$5&3ge0TDT;Sh8vM`_Zu+D|UP z_0)$i?q%EZY6+Ng@sW}^KMoZPVK-2xW=*sm^MAZS0(nb_`507S@E#9xdd0?z^i zg+J6(<|7LP3k}mOA$WE)*zH~2##lzUxkZU^qg&XrU4V1; zby`^7xS&lfxL-kmMG!JC!VseD&fAYa5h%Cm-LLow5`KvVoNVMPvXpb+8SEn%2!|H~ z({&#xJoCX5e|1bTmy%B^SD!TcEy-T$^sAu#U;Qg~+L|e4jKcL~>EOQPKJBOsf!@#r zQJjXS$h_2FR4eW-eJ`g*0i3q()t^ydRNefck&Zn^vlGlu&D6T@985jo&m8wDSOVK9+eQH!y>iaPm+@4a z0?$ra_^bFX>jBCslSZakR%_6V`7ZW27MU$x!0ES z=lJ1`M`;+D|KXy%?ZP`Qi&#>dK1<5n9X=UeT$PaC^)=MNT<^;uCD`%gN7riuuLW?% zN0wKrSYh^@U)~~=VRQr@Ch86)d!?Yh2`ljOn9OrKA9aHU$ZCfn2W=4dii6uE?gCCH zd1M?NCH%INr?l=_n)y{L(%cGqO?Of;IJ(L!hq8q);2E9O$8VLk6}>{#O|q_H=2eXw zEz6ps=Ns$T9k2-N`ZsM#E!SD$&)j(J7#0_1h9IS_H8OZx80R5ZjvdC-QK!`Tm&p?T z;-SZ!3^o07tXzr+xGVAytwqKacg=0xECTYu*b1c(su%cA;zGf1G zzbz^AsJ)#*>;tQ$$cfempVr7D^I%b0L#wmJ5hV+5-z#`dC-LN7yoF3K8J z;NH7y4YIQJCn#Vin7Y%k;FEwh3z&ur{5blJSI!BgaenciogTuTPlb;Hvi^H~1Pz(y4RNinBt9`+m@XpSYF#iC?9!sVq^WGRIZET<*C_M|w~)Fy(HI+?)0NQDTW<5c5{=r1q|#Z!6MuI3c3@_r_-Y>_@YBcH&CpwRmC``u8Im0nKJBo4krEfo zv&G1R7NsQ%RZzE$XFRtm_>)^uBZv)2VA|RG?V`+)#Z7{%B`Dn zI>t#6Tv(WPbwBCtauNa)#Ut?Hy?TF#f4ydsNel{b1Bca!5Ypv?nt`Q<5F)v4VTLxT zDXd_+#^3x{G9$OG8PaZ=2(zm#L}oito?D5no%jlw6&P0%0*<(b3ogy8k%#LFQ%2p? z^X%ArpKz$Yt4ro9a#0h7Z~`$%Q$6*kMsC!CBL9$@q@2%On3No&?@SeJ~%d_5>)QE|H|HS9>qKia%K(}dZABHbF zl2N$w>m4A4NdSWO0ze`%fYfBhwXADrj$R+a0!KsS3CGvkCzq-u5sD>4U;uf!_VamE zOpUW^W%TP=$`#-|`1ewB7&xT?l!@*82|$VSBCm1Au-DjZOghzbnYB~yU?BY&*ILr) z=GV8Ikd%O>1mJ=Pl1DIo*d_l@04xgA^@@l6wRXpn3a$W254#13#2boKbKoJn zV)!RQs7!+knzn}U6*A?$xbhwXtV3q}J7_@FC7(#I7b*@+P1d12(u zZ(K4bZg(uH;tG)TaD)Kq+yusKE09-*G_$0ND)Hy8*faB)g#YZVMCq zy7Ne?Fo5J}nm#|}ZG`Ek|b5$A%hnT*|R2yCTx_lN23bU6@Kx4FJx zUUDAbfd3$cw|ws{f{Ic){MLqL zTZ zA39h+)(enm7&tiYTp*ovtc+u9DI|-&)!%xL-bky`m4>2~oUuEzb$%NPv71!(-2#nU z8pCt9_4YR<3u>owg9iawfC|!)8T+zJfCMA=aL&J~!?wxTnGMzcmvq?9HO2>59G%a_ zJ34F&A^wmES6rpTb~=o!d}??;-z}%)U7

+W~b@|6zdC5eAm`dYAhKk{(VHARWlvEZ4+tq$cS{xia0A z#zZa1pcb%WGn>`@{&65;los)G0Wi?@hnZ}j+XC=hf73z`x!K}I)@VKM275m5+$Qk z^l6s>d6)#qB|2;;1)oa7?E@XQbB!@ok?}>0Z_r_z1jr*KKpb-}M`=~;437BQY?-@E zbu&pyP+&|%gH50T$aqrcBXIvKrDi_8FXRl0 z1hB1%>cG{i{H6*i@K@-XwVagQ-j+XR!-KQ7>ad-21S0um!+5I>(^R%!C2@P24%>r4 zu6qH2*Hg!ElHIZN2Bq732>3^O^H8E@p0bWYip_izCK$lVOvVo8z{UW=LFDU@4!;Z! zqBg-0jIE^3@Ih66Gr<)golsd=Prc}a0$HRn9LVXDsK?KbO~^CY84U1z`tRv<^~X7} z&$bh%3Xl%YTjlYUZMhNwuIB(Hc88M+RKM1COwmn>rS;3^X_*Ni$6|#=FhKyH$c6CH zN2OG!M*^@SdS65tT4-gP>s?rPDhMIhMtZIAXEl_ko2g%WAj$ass%}INy3&;`s4Es6 zzd?=LLc1%6JSCbTJO&JE>s!QgELzcKXFPU50wk`h0@ODP52dN8!*(v>3r6?SV0^f0 zyVMXQNPzfM+f_x!AQ$ml*g0jXz|L3%;%Y$lI3kZFmX`n9&RCp4NpQtz&XV&Oz^Q;Z zl?DkXa^OXF$E6$U8}?Fn_Y(CFe?#gchICn=0c}ay*5f(gi8P=%i$(!c=+l#_u8y=j zG^>#ZyVA@1@tTTb?T%$YDBnsaZoPyfevQiVbjmMocSr}IbfmEsJWoH53IWeFEG>W2 z?wIVtTUmPqaL+Nw#B^YIJf(@OTZh;kS0+f&LG8oi!sJUE;)gz$%Fb5lpicGQfV^Z) z#*K|BXjp#p<0-oM*#N2;5{eURlkN|m)Ueq5Jhiq>=9LVCw zz}7=~A@4LR-AQRb?Ib{s(qLC9avu6{qv=9KHkTTj&!;+laWGJWxrSa=F#*pDgufox zWWuFy2NO>>ULCR@{9=qck3@C5|iR}hJ7DvwB0~0RE8R;czsL*%Qb-f?Rd?cHQBap zuG5k8+cDYdrNPN$jxd(Q{W#!QO!;0y2|XG*%fX>m4TL+k(5L(-->uO3x`kdh0>X9# zv5U$e~*XJSbvv zTCbv;03G?tn9RRMvim-L@FBI9Z=1sSm(@4ytey9G-YQR-cl#LA5Mxj#DC314iQ5&# zX5`ek2(}#r-_o$HXiH8v9xg6JnBql`4inA$#(4~595Niv<=P1bpPHX;Anp{v+PG};D|cmJ%VG> z!*FLZiF-{d19=5(SK_{T+pW0>m-Im>39dZU#P);oJZuDGo Date: Wed, 4 Dec 2019 11:58:11 +0100 Subject: [PATCH 082/257] Store: make index cache shared cache friendly (#1825) * Renamed store cache.go to inmemory.go Signed-off-by: Marco Pracucci * Refactored IndexCache to multiple fetch postings and series within a single call Signed-off-by: Marco Pracucci * Added missing comments to exported functions from InMemoryIndexCache Signed-off-by: Marco Pracucci --- cmd/thanos/store.go | 2 +- pkg/store/bucket.go | 48 ++- pkg/store/bucket_e2e_test.go | 39 ++- pkg/store/cache/cache.go | 292 +--------------- pkg/store/cache/inmemory.go | 318 ++++++++++++++++++ .../cache/{cache_test.go => inmemory_test.go} | 127 ++++--- 6 files changed, 449 insertions(+), 377 deletions(-) create mode 100644 pkg/store/cache/inmemory.go rename pkg/store/cache/{cache_test.go => inmemory_test.go} (79%) diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 51f4723d39..0a3af11677 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -179,7 +179,7 @@ func runStore( // TODO(bwplotka): Add as a flag? maxItemSizeBytes := indexCacheSizeBytes / 2 - indexCache, err := storecache.NewIndexCache(logger, reg, storecache.Opts{ + indexCache, err := storecache.NewInMemoryIndexCache(logger, reg, storecache.Opts{ MaxSizeBytes: indexCacheSizeBytes, MaxItemSizeBytes: maxItemSizeBytes, }) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index e3b65f81b1..c45c8c973a 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -36,6 +36,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/pool" "github.com/thanos-io/thanos/pkg/runutil" + storecache "github.com/thanos-io/thanos/pkg/store/cache" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/strutil" "github.com/thanos-io/thanos/pkg/tracing" @@ -188,13 +189,6 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { return &m } -type indexCache interface { - SetPostings(b ulid.ULID, l labels.Label, v []byte) - Postings(b ulid.ULID, l labels.Label) ([]byte, bool) - SetSeries(b ulid.ULID, id uint64, v []byte) - Series(b ulid.ULID, id uint64) ([]byte, bool) -} - // FilterConfig is a configuration, which Store uses for filtering metrics. type FilterConfig struct { MinTime, MaxTime model.TimeOrDurationValue @@ -207,7 +201,7 @@ type BucketStore struct { metrics *bucketStoreMetrics bucket objstore.BucketReader dir string - indexCache indexCache + indexCache storecache.IndexCache chunkPool *pool.BytesPool // Sets of blocks that have the same labels. They are indexed by a hash over their label set. @@ -241,7 +235,7 @@ func NewBucketStore( reg prometheus.Registerer, bucket objstore.BucketReader, dir string, - indexCache indexCache, + indexCache storecache.IndexCache, maxChunkPoolBytes uint64, maxSampleCount uint64, maxConcurrent int, @@ -1175,7 +1169,7 @@ type bucketBlock struct { bucket objstore.BucketReader meta *metadata.Meta dir string - indexCache indexCache + indexCache storecache.IndexCache chunkPool *pool.BytesPool indexVersion int @@ -1196,7 +1190,7 @@ func newBucketBlock( meta *metadata.Meta, bkt objstore.BucketReader, dir string, - indexCache indexCache, + indexCache storecache.IndexCache, chunkPool *pool.BytesPool, p partitioner, ) (b *bucketBlock, err error) { @@ -1358,13 +1352,13 @@ type bucketIndexReader struct { block *bucketBlock dec *index.Decoder stats *queryStats - cache indexCache + cache storecache.IndexCache mtx sync.Mutex loadedSeries map[uint64][]byte } -func newBucketIndexReader(ctx context.Context, logger log.Logger, block *bucketBlock, cache indexCache) *bucketIndexReader { +func newBucketIndexReader(ctx context.Context, logger log.Logger, block *bucketBlock, cache storecache.IndexCache) *bucketIndexReader { r := &bucketIndexReader{ logger: logger, ctx: ctx, @@ -1527,13 +1521,21 @@ type postingPtr struct { func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { var ptrs []postingPtr + // Fetch postings from the cache with a single call. + keys := make([]labels.Label, 0) + for _, g := range groups { + keys = append(keys, g.keys...) + } + + fromCache, _ := r.cache.FetchMultiPostings(r.block.meta.ULID, keys) + // Iterate over all groups and fetch posting from cache. // If we have a miss, mark key to be fetched in `ptrs` slice. // Overlaps are well handled by partitioner, so we don't need to deduplicate keys. for i, g := range groups { for j, key := range g.keys { // Get postings for the given key from cache first. - if b, ok := r.cache.Postings(r.block.meta.ULID, key); ok { + if b, ok := fromCache[key]; ok { r.stats.postingsTouched++ r.stats.postingsTouchedSizeSum += len(b) @@ -1604,7 +1606,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { // Return postings and fill LRU cache. groups[p.groupID].Fill(p.keyID, fetchedPostings) - r.cache.SetPostings(r.block.meta.ULID, groups[p.groupID].keys[p.keyID], c) + r.cache.StorePostings(r.block.meta.ULID, groups[p.groupID].keys[p.keyID], c) // If we just fetched it we still have to update the stats for touched postings. r.stats.postingsTouched++ @@ -1620,16 +1622,12 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { func (r *bucketIndexReader) PreloadSeries(ids []uint64) error { const maxSeriesSize = 64 * 1024 - var newIDs []uint64 - - for _, id := range ids { - if b, ok := r.cache.Series(r.block.meta.ULID, id); ok { - r.loadedSeries[id] = b - continue - } - newIDs = append(newIDs, id) + // Load series from cache, overwriting the list of ids to preload + // with the missing ones. + fromCache, ids := r.cache.FetchMultiSeries(r.block.meta.ULID, ids) + for id, b := range fromCache { + r.loadedSeries[id] = b } - ids = newIDs parts := r.block.partitioner.Partition(len(ids), func(i int) (start, end uint64) { return ids[i], ids[i] + maxSeriesSize @@ -1674,7 +1672,7 @@ func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, start, } c = c[n : n+int(l)] r.loadedSeries[id] = c - r.cache.SetSeries(r.block.meta.ULID, id, c) + r.cache.StoreSeries(r.block.meta.ULID, id, c) } return nil } diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index d33f530e9b..c30ec4e36d 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -37,33 +37,38 @@ var ( type noopCache struct{} -func (noopCache) SetPostings(b ulid.ULID, l labels.Label, v []byte) {} -func (noopCache) Postings(b ulid.ULID, l labels.Label) ([]byte, bool) { return nil, false } -func (noopCache) SetSeries(b ulid.ULID, id uint64, v []byte) {} -func (noopCache) Series(b ulid.ULID, id uint64) ([]byte, bool) { return nil, false } +func (noopCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte) {} +func (noopCache) FetchMultiPostings(blockID ulid.ULID, keys []labels.Label) (map[labels.Label][]byte, []labels.Label) { + return map[labels.Label][]byte{}, keys +} + +func (noopCache) StoreSeries(blockID ulid.ULID, id uint64, v []byte) {} +func (noopCache) FetchMultiSeries(blockID ulid.ULID, ids []uint64) (map[uint64][]byte, []uint64) { + return map[uint64][]byte{}, ids +} type swappableCache struct { - ptr indexCache + ptr storecache.IndexCache } -func (c *swappableCache) SwapWith(ptr2 indexCache) { +func (c *swappableCache) SwapWith(ptr2 storecache.IndexCache) { c.ptr = ptr2 } -func (c *swappableCache) SetPostings(b ulid.ULID, l labels.Label, v []byte) { - c.ptr.SetPostings(b, l, v) +func (c *swappableCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte) { + c.ptr.StorePostings(blockID, l, v) } -func (c *swappableCache) Postings(b ulid.ULID, l labels.Label) ([]byte, bool) { - return c.ptr.Postings(b, l) +func (c *swappableCache) FetchMultiPostings(blockID ulid.ULID, keys []labels.Label) (map[labels.Label][]byte, []labels.Label) { + return c.ptr.FetchMultiPostings(blockID, keys) } -func (c *swappableCache) SetSeries(b ulid.ULID, id uint64, v []byte) { - c.ptr.SetSeries(b, id, v) +func (c *swappableCache) StoreSeries(blockID ulid.ULID, id uint64, v []byte) { + c.ptr.StoreSeries(blockID, id, v) } -func (c *swappableCache) Series(b ulid.ULID, id uint64) ([]byte, bool) { - return c.ptr.Series(b, id) +func (c *swappableCache) FetchMultiSeries(blockID ulid.ULID, ids []uint64) (map[uint64][]byte, []uint64) { + return c.ptr.FetchMultiSeries(blockID, ids) } type storeSuite struct { @@ -373,7 +378,7 @@ func TestBucketStore_e2e(t *testing.T) { testBucketStore_e2e(t, ctx, s) t.Log("Test with large, sufficient index cache") - indexCache, err := storecache.NewIndexCache(s.logger, nil, storecache.Opts{ + indexCache, err := storecache.NewInMemoryIndexCache(s.logger, nil, storecache.Opts{ MaxItemSizeBytes: 1e5, MaxSizeBytes: 2e5, }) @@ -382,7 +387,7 @@ func TestBucketStore_e2e(t *testing.T) { testBucketStore_e2e(t, ctx, s) t.Log("Test with small index cache") - indexCache2, err := storecache.NewIndexCache(s.logger, nil, storecache.Opts{ + indexCache2, err := storecache.NewInMemoryIndexCache(s.logger, nil, storecache.Opts{ MaxItemSizeBytes: 50, MaxSizeBytes: 100, }) @@ -416,7 +421,7 @@ func TestBucketStore_ManyParts_e2e(t *testing.T) { s := prepareStoreWithTestBlocks(t, dir, bkt, true, 0, emptyRelabelConfig) - indexCache, err := storecache.NewIndexCache(s.logger, nil, storecache.Opts{ + indexCache, err := storecache.NewInMemoryIndexCache(s.logger, nil, storecache.Opts{ MaxItemSizeBytes: 1e5, MaxSizeBytes: 2e5, }) diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index ed4f33cb29..d2c42da88b 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -1,292 +1,22 @@ package storecache import ( - "math" - "sync" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - lru "github.com/hashicorp/golang-lru/simplelru" "github.com/oklog/ulid" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" ) -const ( - cacheTypePostings string = "Postings" - cacheTypeSeries string = "Series" - - sliceHeaderSize = 16 -) - -type cacheKey struct { - block ulid.ULID - key interface{} -} - -func (c cacheKey) keyType() string { - switch c.key.(type) { - case cacheKeyPostings: - return cacheTypePostings - case cacheKeySeries: - return cacheTypeSeries - } - return "" -} - -func (c cacheKey) size() uint64 { - switch k := c.key.(type) { - case cacheKeyPostings: - // ULID + 2 slice headers + number of chars in value and name. - return 16 + 2*sliceHeaderSize + uint64(len(k.Value)+len(k.Name)) - case cacheKeySeries: - return 16 + 8 // ULID + uint64. - } - return 0 -} - -type cacheKeyPostings labels.Label -type cacheKeySeries uint64 - -type IndexCache struct { - mtx sync.Mutex - - logger log.Logger - lru *lru.LRU - maxSizeBytes uint64 - maxItemSizeBytes uint64 - - curSize uint64 - - evicted *prometheus.CounterVec - requests *prometheus.CounterVec - hits *prometheus.CounterVec - added *prometheus.CounterVec - current *prometheus.GaugeVec - currentSize *prometheus.GaugeVec - totalCurrentSize *prometheus.GaugeVec - overflow *prometheus.CounterVec -} - -type Opts struct { - // MaxSizeBytes represents overall maximum number of bytes cache can contain. - MaxSizeBytes uint64 - // MaxItemSizeBytes represents maximum size of single item. - MaxItemSizeBytes uint64 -} - -// NewIndexCache creates a new thread-safe LRU cache for index entries and ensures the total cache -// size approximately does not exceed maxBytes. -func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*IndexCache, error) { - if opts.MaxItemSizeBytes > opts.MaxSizeBytes { - return nil, errors.Errorf("max item size (%v) cannot be bigger than overall cache size (%v)", opts.MaxItemSizeBytes, opts.MaxSizeBytes) - } - - c := &IndexCache{ - logger: logger, - maxSizeBytes: opts.MaxSizeBytes, - maxItemSizeBytes: opts.MaxItemSizeBytes, - } - - c.evicted = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_items_evicted_total", - Help: "Total number of items that were evicted from the index cache.", - }, []string{"item_type"}) - c.evicted.WithLabelValues(cacheTypePostings) - c.evicted.WithLabelValues(cacheTypeSeries) - - c.added = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_items_added_total", - Help: "Total number of items that were added to the index cache.", - }, []string{"item_type"}) - c.added.WithLabelValues(cacheTypePostings) - c.added.WithLabelValues(cacheTypeSeries) - - c.requests = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_requests_total", - Help: "Total number of requests to the cache.", - }, []string{"item_type"}) - c.requests.WithLabelValues(cacheTypePostings) - c.requests.WithLabelValues(cacheTypeSeries) - - c.overflow = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_items_overflowed_total", - Help: "Total number of items that could not be added to the cache due to being too big.", - }, []string{"item_type"}) - c.overflow.WithLabelValues(cacheTypePostings) - c.overflow.WithLabelValues(cacheTypeSeries) - - c.hits = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_hits_total", - Help: "Total number of requests to the cache that were a hit.", - }, []string{"item_type"}) - c.hits.WithLabelValues(cacheTypePostings) - c.hits.WithLabelValues(cacheTypeSeries) - - c.current = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_items", - Help: "Current number of items in the index cache.", - }, []string{"item_type"}) - c.current.WithLabelValues(cacheTypePostings) - c.current.WithLabelValues(cacheTypeSeries) - - c.currentSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_items_size_bytes", - Help: "Current byte size of items in the index cache.", - }, []string{"item_type"}) - c.currentSize.WithLabelValues(cacheTypePostings) - c.currentSize.WithLabelValues(cacheTypeSeries) - - c.totalCurrentSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_total_size_bytes", - Help: "Current byte size of items (both value and key) in the index cache.", - }, []string{"item_type"}) - c.totalCurrentSize.WithLabelValues(cacheTypePostings) - c.totalCurrentSize.WithLabelValues(cacheTypeSeries) - - if reg != nil { - reg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_max_size_bytes", - Help: "Maximum number of bytes to be held in the index cache.", - }, func() float64 { - return float64(c.maxSizeBytes) - })) - reg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_max_item_size_bytes", - Help: "Maximum number of bytes for single entry to be held in the index cache.", - }, func() float64 { - return float64(c.maxItemSizeBytes) - })) - reg.MustRegister(c.requests, c.hits, c.added, c.evicted, c.current, c.currentSize, c.totalCurrentSize, c.overflow) - } +type IndexCache interface { + // StorePostings stores postings for a single series. + StorePostings(blockID ulid.ULID, l labels.Label, v []byte) - // Initialize LRU cache with a high size limit since we will manage evictions ourselves - // based on stored size using `RemoveOldest` method. - l, err := lru.NewLRU(math.MaxInt64, c.onEvict) - if err != nil { - return nil, err - } - c.lru = l + // FetchMultiPostings fetches multiple postings - each identified by a label - + // and returns a map containing cache hits, along with a list of missing keys. + FetchMultiPostings(blockID ulid.ULID, keys []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) - level.Info(logger).Log( - "msg", "created index cache", - "maxItemSizeBytes", c.maxItemSizeBytes, - "maxSizeBytes", c.maxSizeBytes, - "maxItems", "math.MaxInt64", - ) - return c, nil -} - -func (c *IndexCache) onEvict(key, val interface{}) { - k := key.(cacheKey).keyType() - entrySize := sliceHeaderSize + uint64(len(val.([]byte))) - - c.evicted.WithLabelValues(string(k)).Inc() - c.current.WithLabelValues(string(k)).Dec() - c.currentSize.WithLabelValues(string(k)).Sub(float64(entrySize)) - c.totalCurrentSize.WithLabelValues(string(k)).Sub(float64(entrySize + key.(cacheKey).size())) - - c.curSize -= entrySize -} - -func (c *IndexCache) get(typ string, key cacheKey) ([]byte, bool) { - c.requests.WithLabelValues(typ).Inc() - - c.mtx.Lock() - defer c.mtx.Unlock() - - v, ok := c.lru.Get(key) - if !ok { - return nil, false - } - c.hits.WithLabelValues(typ).Inc() - return v.([]byte), true -} - -func (c *IndexCache) set(typ string, key cacheKey, val []byte) { - var size = sliceHeaderSize + uint64(len(val)) - - c.mtx.Lock() - defer c.mtx.Unlock() - - if _, ok := c.lru.Get(key); ok { - return - } - - if !c.ensureFits(size, typ) { - c.overflow.WithLabelValues(typ).Inc() - return - } - - // The caller may be passing in a sub-slice of a huge array. Copy the data - // to ensure we don't waste huge amounts of space for something small. - v := make([]byte, len(val)) - copy(v, val) - c.lru.Add(key, v) - - c.added.WithLabelValues(typ).Inc() - c.currentSize.WithLabelValues(typ).Add(float64(size)) - c.totalCurrentSize.WithLabelValues(typ).Add(float64(size + key.size())) - c.current.WithLabelValues(typ).Inc() - c.curSize += size -} - -// ensureFits tries to make sure that the passed slice will fit into the LRU cache. -// Returns true if it will fit. -func (c *IndexCache) ensureFits(size uint64, typ string) bool { - if size > c.maxItemSizeBytes { - level.Debug(c.logger).Log( - "msg", "item bigger than maxItemSizeBytes. Ignoring..", - "maxItemSizeBytes", c.maxItemSizeBytes, - "maxSizeBytes", c.maxSizeBytes, - "curSize", c.curSize, - "itemSize", size, - "cacheType", typ, - ) - return false - } - - for c.curSize+size > c.maxSizeBytes { - if _, _, ok := c.lru.RemoveOldest(); !ok { - level.Error(c.logger).Log( - "msg", "LRU has nothing more to evict, but we still cannot allocate the item. Resetting cache.", - "maxItemSizeBytes", c.maxItemSizeBytes, - "maxSizeBytes", c.maxSizeBytes, - "curSize", c.curSize, - "itemSize", size, - "cacheType", typ, - ) - c.reset() - } - } - return true -} - -func (c *IndexCache) reset() { - c.lru.Purge() - c.current.Reset() - c.currentSize.Reset() - c.totalCurrentSize.Reset() - c.curSize = 0 -} - -// SetPostings sets the postings identfied by the ulid and label to the value v, -// if the postings already exists in the cache it is not mutated. -func (c *IndexCache) SetPostings(b ulid.ULID, l labels.Label, v []byte) { - c.set(cacheTypePostings, cacheKey{b, cacheKeyPostings(l)}, v) -} - -func (c *IndexCache) Postings(b ulid.ULID, l labels.Label) ([]byte, bool) { - return c.get(cacheTypePostings, cacheKey{b, cacheKeyPostings(l)}) -} - -// SetSeries sets the series identfied by the ulid and id to the value v, -// if the series already exists in the cache it is not mutated. -func (c *IndexCache) SetSeries(b ulid.ULID, id uint64, v []byte) { - c.set(cacheTypeSeries, cacheKey{b, cacheKeySeries(id)}, v) -} + // StoreSeries stores a single series. + StoreSeries(blockID ulid.ULID, id uint64, v []byte) -func (c *IndexCache) Series(b ulid.ULID, id uint64) ([]byte, bool) { - return c.get(cacheTypeSeries, cacheKey{b, cacheKeySeries(id)}) + // FetchMultiSeries fetches multiple series - each identified by ID - from the cache + // and returns a map containing cache hits, along with a list of missing IDs. + FetchMultiSeries(blockID ulid.ULID, ids []uint64) (hits map[uint64][]byte, misses []uint64) } diff --git a/pkg/store/cache/inmemory.go b/pkg/store/cache/inmemory.go new file mode 100644 index 0000000000..3ef7d0c7ce --- /dev/null +++ b/pkg/store/cache/inmemory.go @@ -0,0 +1,318 @@ +package storecache + +import ( + "math" + "sync" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + lru "github.com/hashicorp/golang-lru/simplelru" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/labels" +) + +const ( + cacheTypePostings string = "Postings" + cacheTypeSeries string = "Series" + + sliceHeaderSize = 16 +) + +type cacheKey struct { + block ulid.ULID + key interface{} +} + +func (c cacheKey) keyType() string { + switch c.key.(type) { + case cacheKeyPostings: + return cacheTypePostings + case cacheKeySeries: + return cacheTypeSeries + } + return "" +} + +func (c cacheKey) size() uint64 { + switch k := c.key.(type) { + case cacheKeyPostings: + // ULID + 2 slice headers + number of chars in value and name. + return 16 + 2*sliceHeaderSize + uint64(len(k.Value)+len(k.Name)) + case cacheKeySeries: + return 16 + 8 // ULID + uint64. + } + return 0 +} + +type cacheKeyPostings labels.Label +type cacheKeySeries uint64 + +type InMemoryIndexCache struct { + mtx sync.Mutex + + logger log.Logger + lru *lru.LRU + maxSizeBytes uint64 + maxItemSizeBytes uint64 + + curSize uint64 + + evicted *prometheus.CounterVec + requests *prometheus.CounterVec + hits *prometheus.CounterVec + added *prometheus.CounterVec + current *prometheus.GaugeVec + currentSize *prometheus.GaugeVec + totalCurrentSize *prometheus.GaugeVec + overflow *prometheus.CounterVec +} + +type Opts struct { + // MaxSizeBytes represents overall maximum number of bytes cache can contain. + MaxSizeBytes uint64 + // MaxItemSizeBytes represents maximum size of single item. + MaxItemSizeBytes uint64 +} + +// NewInMemoryIndexCache creates a new thread-safe LRU cache for index entries and ensures the total cache +// size approximately does not exceed maxBytes. +func NewInMemoryIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*InMemoryIndexCache, error) { + if opts.MaxItemSizeBytes > opts.MaxSizeBytes { + return nil, errors.Errorf("max item size (%v) cannot be bigger than overall cache size (%v)", opts.MaxItemSizeBytes, opts.MaxSizeBytes) + } + + c := &InMemoryIndexCache{ + logger: logger, + maxSizeBytes: opts.MaxSizeBytes, + maxItemSizeBytes: opts.MaxItemSizeBytes, + } + + c.evicted = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_items_evicted_total", + Help: "Total number of items that were evicted from the index cache.", + }, []string{"item_type"}) + c.evicted.WithLabelValues(cacheTypePostings) + c.evicted.WithLabelValues(cacheTypeSeries) + + c.added = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_items_added_total", + Help: "Total number of items that were added to the index cache.", + }, []string{"item_type"}) + c.added.WithLabelValues(cacheTypePostings) + c.added.WithLabelValues(cacheTypeSeries) + + c.requests = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_requests_total", + Help: "Total number of requests to the cache.", + }, []string{"item_type"}) + c.requests.WithLabelValues(cacheTypePostings) + c.requests.WithLabelValues(cacheTypeSeries) + + c.overflow = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_items_overflowed_total", + Help: "Total number of items that could not be added to the cache due to being too big.", + }, []string{"item_type"}) + c.overflow.WithLabelValues(cacheTypePostings) + c.overflow.WithLabelValues(cacheTypeSeries) + + c.hits = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_hits_total", + Help: "Total number of requests to the cache that were a hit.", + }, []string{"item_type"}) + c.hits.WithLabelValues(cacheTypePostings) + c.hits.WithLabelValues(cacheTypeSeries) + + c.current = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_items", + Help: "Current number of items in the index cache.", + }, []string{"item_type"}) + c.current.WithLabelValues(cacheTypePostings) + c.current.WithLabelValues(cacheTypeSeries) + + c.currentSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_items_size_bytes", + Help: "Current byte size of items in the index cache.", + }, []string{"item_type"}) + c.currentSize.WithLabelValues(cacheTypePostings) + c.currentSize.WithLabelValues(cacheTypeSeries) + + c.totalCurrentSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_total_size_bytes", + Help: "Current byte size of items (both value and key) in the index cache.", + }, []string{"item_type"}) + c.totalCurrentSize.WithLabelValues(cacheTypePostings) + c.totalCurrentSize.WithLabelValues(cacheTypeSeries) + + if reg != nil { + reg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_max_size_bytes", + Help: "Maximum number of bytes to be held in the index cache.", + }, func() float64 { + return float64(c.maxSizeBytes) + })) + reg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_max_item_size_bytes", + Help: "Maximum number of bytes for single entry to be held in the index cache.", + }, func() float64 { + return float64(c.maxItemSizeBytes) + })) + reg.MustRegister(c.requests, c.hits, c.added, c.evicted, c.current, c.currentSize, c.totalCurrentSize, c.overflow) + } + + // Initialize LRU cache with a high size limit since we will manage evictions ourselves + // based on stored size using `RemoveOldest` method. + l, err := lru.NewLRU(math.MaxInt64, c.onEvict) + if err != nil { + return nil, err + } + c.lru = l + + level.Info(logger).Log( + "msg", "created index cache", + "maxItemSizeBytes", c.maxItemSizeBytes, + "maxSizeBytes", c.maxSizeBytes, + "maxItems", "math.MaxInt64", + ) + return c, nil +} + +func (c *InMemoryIndexCache) onEvict(key, val interface{}) { + k := key.(cacheKey).keyType() + entrySize := sliceHeaderSize + uint64(len(val.([]byte))) + + c.evicted.WithLabelValues(string(k)).Inc() + c.current.WithLabelValues(string(k)).Dec() + c.currentSize.WithLabelValues(string(k)).Sub(float64(entrySize)) + c.totalCurrentSize.WithLabelValues(string(k)).Sub(float64(entrySize + key.(cacheKey).size())) + + c.curSize -= entrySize +} + +func (c *InMemoryIndexCache) get(typ string, key cacheKey) ([]byte, bool) { + c.requests.WithLabelValues(typ).Inc() + + c.mtx.Lock() + defer c.mtx.Unlock() + + v, ok := c.lru.Get(key) + if !ok { + return nil, false + } + c.hits.WithLabelValues(typ).Inc() + return v.([]byte), true +} + +func (c *InMemoryIndexCache) set(typ string, key cacheKey, val []byte) { + var size = sliceHeaderSize + uint64(len(val)) + + c.mtx.Lock() + defer c.mtx.Unlock() + + if _, ok := c.lru.Get(key); ok { + return + } + + if !c.ensureFits(size, typ) { + c.overflow.WithLabelValues(typ).Inc() + return + } + + // The caller may be passing in a sub-slice of a huge array. Copy the data + // to ensure we don't waste huge amounts of space for something small. + v := make([]byte, len(val)) + copy(v, val) + c.lru.Add(key, v) + + c.added.WithLabelValues(typ).Inc() + c.currentSize.WithLabelValues(typ).Add(float64(size)) + c.totalCurrentSize.WithLabelValues(typ).Add(float64(size + key.size())) + c.current.WithLabelValues(typ).Inc() + c.curSize += size +} + +// ensureFits tries to make sure that the passed slice will fit into the LRU cache. +// Returns true if it will fit. +func (c *InMemoryIndexCache) ensureFits(size uint64, typ string) bool { + if size > c.maxItemSizeBytes { + level.Debug(c.logger).Log( + "msg", "item bigger than maxItemSizeBytes. Ignoring..", + "maxItemSizeBytes", c.maxItemSizeBytes, + "maxSizeBytes", c.maxSizeBytes, + "curSize", c.curSize, + "itemSize", size, + "cacheType", typ, + ) + return false + } + + for c.curSize+size > c.maxSizeBytes { + if _, _, ok := c.lru.RemoveOldest(); !ok { + level.Error(c.logger).Log( + "msg", "LRU has nothing more to evict, but we still cannot allocate the item. Resetting cache.", + "maxItemSizeBytes", c.maxItemSizeBytes, + "maxSizeBytes", c.maxSizeBytes, + "curSize", c.curSize, + "itemSize", size, + "cacheType", typ, + ) + c.reset() + } + } + return true +} + +func (c *InMemoryIndexCache) reset() { + c.lru.Purge() + c.current.Reset() + c.currentSize.Reset() + c.totalCurrentSize.Reset() + c.curSize = 0 +} + +// StorePostings sets the postings identified by the ulid and label to the value v, +// if the postings already exists in the cache it is not mutated. +func (c *InMemoryIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte) { + c.set(cacheTypePostings, cacheKey{blockID, cacheKeyPostings(l)}, v) +} + +// FetchMultiPostings fetches multiple postings - each identified by a label - +// and returns a map containing cache hits, along with a list of missing keys. +func (c *InMemoryIndexCache) FetchMultiPostings(blockID ulid.ULID, keys []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) { + hits = map[labels.Label][]byte{} + + for _, key := range keys { + if b, ok := c.get(cacheTypePostings, cacheKey{blockID, cacheKeyPostings(key)}); ok { + hits[key] = b + continue + } + + misses = append(misses, key) + } + + return hits, misses +} + +// StoreSeries sets the series identified by the ulid and id to the value v, +// if the series already exists in the cache it is not mutated. +func (c *InMemoryIndexCache) StoreSeries(blockID ulid.ULID, id uint64, v []byte) { + c.set(cacheTypeSeries, cacheKey{blockID, cacheKeySeries(id)}, v) +} + +// FetchMultiSeries fetches multiple series - each identified by ID - from the cache +// and returns a map containing cache hits, along with a list of missing IDs. +func (c *InMemoryIndexCache) FetchMultiSeries(blockID ulid.ULID, ids []uint64) (hits map[uint64][]byte, misses []uint64) { + hits = map[uint64][]byte{} + + for _, id := range ids { + if b, ok := c.get(cacheTypeSeries, cacheKey{blockID, cacheKeySeries(id)}); ok { + hits[id] = b + continue + } + + misses = append(misses, id) + } + + return hits, misses +} diff --git a/pkg/store/cache/cache_test.go b/pkg/store/cache/inmemory_test.go similarity index 79% rename from pkg/store/cache/cache_test.go rename to pkg/store/cache/inmemory_test.go index 3351543cfd..f70c79bf50 100644 --- a/pkg/store/cache/cache_test.go +++ b/pkg/store/cache/inmemory_test.go @@ -18,11 +18,11 @@ import ( "github.com/thanos-io/thanos/pkg/testutil" ) -func TestIndexCache_AvoidsDeadlock(t *testing.T) { +func TestInMemoryIndexCache_AvoidsDeadlock(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() metrics := prometheus.NewRegistry() - cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ + cache, err := NewInMemoryIndexCache(log.NewNopLogger(), metrics, Opts{ MaxItemSizeBytes: sliceHeaderSize + 5, MaxSizeBytes: sliceHeaderSize + 5, }) @@ -37,21 +37,21 @@ func TestIndexCache_AvoidsDeadlock(t *testing.T) { testutil.Ok(t, err) cache.lru = l - cache.SetPostings(ulid.MustNew(0, nil), labels.Label{Name: "test2", Value: "1"}, []byte{42, 33, 14, 67, 11}) + cache.StorePostings(ulid.MustNew(0, nil), labels.Label{Name: "test2", Value: "1"}, []byte{42, 33, 14, 67, 11}) testutil.Equals(t, uint64(sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(cache.curSize), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) // This triggers deadlock logic. - cache.SetPostings(ulid.MustNew(0, nil), labels.Label{Name: "test1", Value: "1"}, []byte{42}) + cache.StorePostings(ulid.MustNew(0, nil), labels.Label{Name: "test1", Value: "1"}, []byte{42}) testutil.Equals(t, uint64(sliceHeaderSize+1), cache.curSize) testutil.Equals(t, float64(cache.curSize), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) } -func TestIndexCache_UpdateItem(t *testing.T) { +func TestInMemoryIndexCache_UpdateItem(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() const maxSize = 2 * (sliceHeaderSize + 1) @@ -74,7 +74,7 @@ func TestIndexCache_UpdateItem(t *testing.T) { }) metrics := prometheus.NewRegistry() - cache, err := NewIndexCache(log.NewSyncLogger(errorLogger), metrics, Opts{ + cache, err := NewInMemoryIndexCache(log.NewSyncLogger(errorLogger), metrics, Opts{ MaxItemSizeBytes: maxSize, MaxSizeBytes: maxSize, }) @@ -90,13 +90,23 @@ func TestIndexCache_UpdateItem(t *testing.T) { }{ { typ: cacheTypePostings, - set: func(id uint64, b []byte) { cache.SetPostings(uid(id), lbl, b) }, - get: func(id uint64) ([]byte, bool) { return cache.Postings(uid(id), lbl) }, + set: func(id uint64, b []byte) { cache.StorePostings(uid(id), lbl, b) }, + get: func(id uint64) ([]byte, bool) { + hits, _ := cache.FetchMultiPostings(uid(id), []labels.Label{lbl}) + b, ok := hits[lbl] + + return b, ok + }, }, { typ: cacheTypeSeries, - set: func(id uint64, b []byte) { cache.SetSeries(uid(id), id, b) }, - get: func(id uint64) ([]byte, bool) { return cache.Series(uid(id), id) }, + set: func(id uint64, b []byte) { cache.StoreSeries(uid(id), id, b) }, + get: func(id uint64) ([]byte, bool) { + hits, _ := cache.FetchMultiSeries(uid(id), []uint64{id}) + b, ok := hits[id] + + return b, ok + }, }, } { t.Run(tt.typ, func(t *testing.T) { @@ -145,11 +155,11 @@ func TestIndexCache_UpdateItem(t *testing.T) { } // This should not happen as we hardcode math.MaxInt, but we still add test to check this out. -func TestIndexCache_MaxNumberOfItemsHit(t *testing.T) { +func TestInMemoryIndexCache_MaxNumberOfItemsHit(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() metrics := prometheus.NewRegistry() - cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ + cache, err := NewInMemoryIndexCache(log.NewNopLogger(), metrics, Opts{ MaxItemSizeBytes: 2*sliceHeaderSize + 10, MaxSizeBytes: 2*sliceHeaderSize + 10, }) @@ -161,9 +171,9 @@ func TestIndexCache_MaxNumberOfItemsHit(t *testing.T) { id := ulid.MustNew(0, nil) - cache.SetPostings(id, labels.Label{Name: "test", Value: "123"}, []byte{42, 33}) - cache.SetPostings(id, labels.Label{Name: "test", Value: "124"}, []byte{42, 33}) - cache.SetPostings(id, labels.Label{Name: "test", Value: "125"}, []byte{42, 33}) + cache.StorePostings(id, labels.Label{Name: "test", Value: "123"}, []byte{42, 33}) + cache.StorePostings(id, labels.Label{Name: "test", Value: "124"}, []byte{42, 33}) + cache.StorePostings(id, labels.Label{Name: "test", Value: "125"}, []byte{42, 33}) testutil.Equals(t, uint64(2*sliceHeaderSize+4), cache.curSize) testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) @@ -178,11 +188,11 @@ func TestIndexCache_MaxNumberOfItemsHit(t *testing.T) { testutil.Equals(t, float64(0), promtest.ToFloat64(cache.hits.WithLabelValues(cacheTypeSeries))) } -func TestIndexCache_Eviction_WithMetrics(t *testing.T) { +func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() metrics := prometheus.NewRegistry() - cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ + cache, err := NewInMemoryIndexCache(log.NewNopLogger(), metrics, Opts{ MaxItemSizeBytes: 2*sliceHeaderSize + 5, MaxSizeBytes: 2*sliceHeaderSize + 5, }) @@ -190,12 +200,17 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { id := ulid.MustNew(0, nil) lbls := labels.Label{Name: "test", Value: "123"} + emptyPostingsHits := map[labels.Label][]byte{} + emptyPostingsMisses := []labels.Label(nil) + emptySeriesHits := map[uint64][]byte{} + emptySeriesMisses := []uint64(nil) - _, ok := cache.Postings(id, lbls) - testutil.Assert(t, !ok, "no such key") + pHits, pMisses := cache.FetchMultiPostings(id, []labels.Label{lbls}) + testutil.Equals(t, emptyPostingsHits, pHits, "no such key") + testutil.Equals(t, []labels.Label{lbls}, pMisses) // Add sliceHeaderSize + 2 bytes. - cache.SetPostings(id, lbls, []byte{42, 33}) + cache.StorePostings(id, lbls, []byte{42, 33}) testutil.Equals(t, uint64(sliceHeaderSize+2), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(sliceHeaderSize+2), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) @@ -208,17 +223,20 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - p, ok := cache.Postings(id, lbls) - testutil.Assert(t, ok, "key exists") - testutil.Equals(t, []byte{42, 33}, p) + pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls}) + testutil.Equals(t, map[labels.Label][]byte{lbls: []byte{42, 33}}, pHits, "key exists") + testutil.Equals(t, emptyPostingsMisses, pMisses) + + pHits, pMisses = cache.FetchMultiPostings(ulid.MustNew(1, nil), []labels.Label{lbls}) + testutil.Equals(t, emptyPostingsHits, pHits, "no such key") + testutil.Equals(t, []labels.Label{lbls}, pMisses) - _, ok = cache.Postings(ulid.MustNew(1, nil), lbls) - testutil.Assert(t, !ok, "no such key") - _, ok = cache.Postings(id, labels.Label{Name: "test", Value: "124"}) - testutil.Assert(t, !ok, "no such key") + pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{{Name: "test", Value: "124"}}) + testutil.Equals(t, emptyPostingsHits, pHits, "no such key") + testutil.Equals(t, []labels.Label{{Name: "test", Value: "124"}}, pMisses) // Add sliceHeaderSize + 3 more bytes. - cache.SetSeries(id, 1234, []byte{222, 223, 224}) + cache.StoreSeries(id, 1234, []byte{222, 223, 224}) testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(sliceHeaderSize+2), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) @@ -231,9 +249,9 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - p, ok = cache.Series(id, 1234) - testutil.Assert(t, ok, "key exists") - testutil.Equals(t, []byte{222, 223, 224}, p) + sHits, sMisses := cache.FetchMultiSeries(id, []uint64{1234}) + testutil.Equals(t, map[uint64][]byte{1234: []byte{222, 223, 224}}, sHits, "key exists") + testutil.Equals(t, emptySeriesMisses, sMisses) lbls2 := labels.Label{Name: "test", Value: "124"} @@ -242,7 +260,7 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { for i := 0; i < sliceHeaderSize; i++ { v = append(v, 3) } - cache.SetPostings(id, lbls2, v) + cache.StorePostings(id, lbls2, v) testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) @@ -257,17 +275,20 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) // Eviction. // Evicted. - _, ok = cache.Postings(id, lbls) - testutil.Assert(t, !ok, "no such key") - _, ok = cache.Series(id, 1234) - testutil.Assert(t, !ok, "no such key") + pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls}) + testutil.Equals(t, emptyPostingsHits, pHits, "no such key") + testutil.Equals(t, []labels.Label{lbls}, pMisses) - p, ok = cache.Postings(id, lbls2) - testutil.Assert(t, ok, "key exists") - testutil.Equals(t, v, p) + sHits, sMisses = cache.FetchMultiSeries(id, []uint64{1234}) + testutil.Equals(t, emptySeriesHits, sHits, "no such key") + testutil.Equals(t, []uint64{1234}, sMisses) + + pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls2}) + testutil.Equals(t, map[labels.Label][]byte{lbls2: v}, pHits) + testutil.Equals(t, emptyPostingsMisses, pMisses) // Add same item again. - cache.SetPostings(id, lbls2, v) + cache.StorePostings(id, lbls2, v) testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) @@ -281,12 +302,12 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - p, ok = cache.Postings(id, lbls2) - testutil.Assert(t, ok, "key exists") - testutil.Equals(t, v, p) + pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls2}) + testutil.Equals(t, map[labels.Label][]byte{lbls2: v}, pHits) + testutil.Equals(t, emptyPostingsMisses, pMisses) // Add too big item. - cache.SetPostings(id, labels.Label{Name: "test", Value: "toobig"}, append(v, 5)) + cache.StorePostings(id, labels.Label{Name: "test", Value: "toobig"}, append(v, 5)) testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(2*sliceHeaderSize+5), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) @@ -299,7 +320,7 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - _, _, ok = cache.lru.RemoveOldest() + _, _, ok := cache.lru.RemoveOldest() testutil.Assert(t, ok, "something to remove") testutil.Equals(t, uint64(0), cache.curSize) @@ -319,7 +340,7 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { lbls3 := labels.Label{Name: "test", Value: "124"} - cache.SetPostings(id, lbls3, []byte{}) + cache.StorePostings(id, lbls3, []byte{}) testutil.Equals(t, uint64(sliceHeaderSize), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) @@ -333,13 +354,13 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(2), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - p, ok = cache.Postings(id, lbls3) - testutil.Assert(t, ok, "key exists") - testutil.Equals(t, []byte{}, p) + pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls3}) + testutil.Equals(t, map[labels.Label][]byte{lbls3: []byte{}}, pHits, "key exists") + testutil.Equals(t, emptyPostingsMisses, pMisses) // nil works and still allocates empty slice. lbls4 := labels.Label{Name: "test", Value: "125"} - cache.SetPostings(id, lbls4, []byte(nil)) + cache.StorePostings(id, lbls4, []byte(nil)) testutil.Equals(t, 2*uint64(sliceHeaderSize), cache.curSize) testutil.Equals(t, float64(2), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) @@ -353,9 +374,9 @@ func TestIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(2), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - p, ok = cache.Postings(id, lbls4) - testutil.Assert(t, ok, "key exists") - testutil.Equals(t, []byte{}, p) + pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls4}) + testutil.Equals(t, map[labels.Label][]byte{lbls4: []byte{}}, pHits, "key exists") + testutil.Equals(t, emptyPostingsMisses, pMisses) // Other metrics. testutil.Equals(t, float64(4), promtest.ToFloat64(cache.added.WithLabelValues(cacheTypePostings))) From a90c52707a8fd7170c57812bcb93081c17bc69ae Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 5 Dec 2019 18:09:02 +0800 Subject: [PATCH 083/257] add deep dive talk (#1840) Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/getting-started.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 1100b4f43e..17519e88ec 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -99,6 +99,7 @@ We also have example Grafana dashboards [here](/examples/grafana/monitoring.md) * 02.2018: [Very first Prometheus Meetup Slides](https://www.slideshare.net/BartomiejPotka/thanos-global-durable-prometheus-monitoring) * 02.2019: [FOSDEM + demo](https://fosdem.org/2019/schedule/event/thanos_transforming_prometheus_to_a_global_scale_in_a_seven_simple_steps/) * 09.2019: [CloudNative Warsaw Slides](https://docs.google.com/presentation/d/1cKpbJY3jIAtr03M-zcNujwBA38_LDj7NqE4LjNfvglE/edit?usp=sharing) +* 11.2019: [CloudNative Deep Dive](https://www.youtube.com/watch?v=qQN0N14HXPM) ## Blog posts From 91937303d080356fd82f2aa52c5bd18f354a8510 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 5 Dec 2019 18:15:42 +0800 Subject: [PATCH 084/257] add k8s comment for service discovery (#1841) Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/service-discovery.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/service-discovery.md b/docs/service-discovery.md index 3931f2f0b1..d9f15074a6 100644 --- a/docs/service-discovery.md +++ b/docs/service-discovery.md @@ -96,6 +96,14 @@ An example using this lookup with a static flag: --store=dnssrv+_thanosstores._tcp.mycompany.org ``` +It is also work in kubernetes cluster. An example: + +``` +--store=dnssrv+_grpc._tcp.thanos-store.monitoring.svc +``` + +It means that in kubernetes cluster, there is a service named "thanos-store" in `monitoring` namespace with a port defined named "grpc". + * `dnssrvnoa+` - the domain name after this prefix will be looked up as a SRV query, with no A/AAAA lookup made after that. Similar to the `dnssrv+` case, you do not need to specify a port. An example: ``` From fd010269ba3faaa4c0f49daaa102875c2f04f4c2 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Thu, 5 Dec 2019 10:16:44 +0000 Subject: [PATCH 085/257] Promote sidecar shipper.upload-compacted flag to non hidden, non experimental state. (#1833) Signed-off-by: Bartek Plotka --- cmd/thanos/sidecar.go | 2 +- docs/components/sidecar.md | 6 ++++++ docs/quick-tutorial.md | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index f4b319898c..859f58ac6d 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -58,7 +58,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { objStoreConfig := regCommonObjStoreFlags(cmd, "", false) - uploadCompacted := cmd.Flag("shipper.upload-compacted", "[Experimental] If true sidecar will try to upload compacted blocks as well. Useful for migration purposes. Works only if compaction is disabled on Prometheus.").Default("false").Hidden().Bool() + uploadCompacted := cmd.Flag("shipper.upload-compacted", "If true sidecar will try to upload compacted blocks as well. Useful for migration purposes. Works only if compaction is disabled on Prometheus. Do it once and then disable the flag when done.").Default("false").Bool() minTime := thanosmodel.TimeOrDuration(cmd.Flag("min-time", "Start of time range limit to serve. Thanos sidecar will serve only metrics, which happened later than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y."). Default("0000-01-01T00:00:00Z")) diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 2d735061f3..0534a59ce2 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -142,6 +142,12 @@ Flags: contains object store configuration. See format details: https://thanos.io/storage.md/#configuration + --shipper.upload-compacted + If true sidecar will try to upload compacted + blocks as well. Useful for migration purposes. + Works only if compaction is disabled on + Prometheus. Do it once and then disable the + flag when done. --min-time=0000-01-01T00:00:00Z Start of time range limit to serve. Thanos sidecar will serve only metrics, which happened diff --git a/docs/quick-tutorial.md b/docs/quick-tutorial.md index 42d51bcdeb..15191c5fdd 100644 --- a/docs/quick-tutorial.md +++ b/docs/quick-tutorial.md @@ -93,6 +93,11 @@ thanos sidecar \ * _[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)_ +### Uploading old metrics. + +When sidecar is run with the `--shipper.upload-compacted` flag it will sync all older existing blocks from the Prometheus local storage on startup. +NOTE: This assumes you never run sidecar with block uploading against this bucket. Otherwise manual steps are needed to remove overlapping blocks from the bucket. +Those will be suggested by the sidecar verification process. #### External Labels From 9afb6d817119def683bba23738594d9f0a28505b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Thu, 5 Dec 2019 11:53:27 +0100 Subject: [PATCH 086/257] docs/gettingstarted.md: add intro talk (#1843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lucas Servén Marín --- docs/getting-started.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 17519e88ec..f263313067 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -100,6 +100,7 @@ We also have example Grafana dashboards [here](/examples/grafana/monitoring.md) * 02.2019: [FOSDEM + demo](https://fosdem.org/2019/schedule/event/thanos_transforming_prometheus_to_a_global_scale_in_a_seven_simple_steps/) * 09.2019: [CloudNative Warsaw Slides](https://docs.google.com/presentation/d/1cKpbJY3jIAtr03M-zcNujwBA38_LDj7NqE4LjNfvglE/edit?usp=sharing) * 11.2019: [CloudNative Deep Dive](https://www.youtube.com/watch?v=qQN0N14HXPM) +* 11.2019: [CloudNative Intro](https://www.youtube.com/watch?v=m0JgWlTc60Q) ## Blog posts From 74efe882016e883f094e92bea40d23bf3ac14bb1 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 5 Dec 2019 20:51:02 +0800 Subject: [PATCH 087/257] comment upload compacted or not (#1814) * comment upload compacted or not Signed-off-by: Xiang Dai <764524258@qq.com> * feedback Signed-off-by: Xiang Dai <764524258@qq.com> * fix nits Signed-off-by: Xiang Dai <764524258@qq.com> * feedback Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/components/sidecar.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 0534a59ce2..4bdd8ced46 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -29,8 +29,10 @@ Prometheus servers connected to the Thanos cluster via the sidecar are subject t * The `--web.enable-admin-api` flag is enabled to support sidecar to get metadata from Prometheus like external labels. * The `--web.enable-lifecycle` flag is enabled if you want to use sidecar reloading features (`--reload.*` flags). -If you choose to use the sidecar to also upload to object storage: +If you choose to use the sidecar to also upload data to object storage: +* Must specify object storage (`--objstore.*` flags) +* It only uploads uncompacted Prometheus blocks. For compacted blocks, see [Upload compacted blocks](./sidecar.md/#upload-compacted-blocks-experimental). * The `--storage.tsdb.min-block-duration` and `--storage.tsdb.max-block-duration` must be set to equal values to disable local compaction on order to use Thanos sidecar upload, otherwise leave local compaction on if sidecar just exposes StoreAPI and your retention is normal. The default of `2h` is recommended. Mentioned parameters set to equal values disable the internal Prometheus compaction, which is needed to avoid the uploaded data corruption when Thanos compactor does its job, this is critical for data consistency and should not be ignored if you plan to use Thanos compactor. Even though you set mentioned parameters equal, you might observe Prometheus internal metric `prometheus_tsdb_compactions_total` being incremented, don't be confused by that: Prometheus writes initial head block to filesytem via its internal compaction mechanism, but if you have followed recommendations - data won't be modified by Prometheus before the sidecar uploads it. Thanos sidecar will also check sanity of the flags set to Prometheus on the startup and log errors or warning if they have been configured improperly (#838). * The retention is recommended to not be lower than three times the min block duration, so 6 hours. This achieves resilience in the face of connectivity issues to the object storage since all local data will remain available within the Thanos cluster. If connectivity gets restored the backlog of blocks gets uploaded to the object storage. @@ -70,7 +72,7 @@ config: ## Upload compacted blocks (EXPERIMENTAL) -If you want to migrate from a pure Prometheus setup to Thanos and have to keep the historical data, you can use the flag `--shipper.upload-compacted`. This will also upload blocks that were compacted by Prometheus. +If you want to migrate from a pure Prometheus setup to Thanos and have to keep the historical data, you can use the flag `--shipper.upload-compacted`. This will also upload blocks that were compacted by Prometheus. Values greater than 1 in the `compaction.level` field of a Prometheus block’s `meta.json` file indicate level of compaction. To use this, the Prometheus compaction needs to be disabled. This can be done by setting the following flags for Prometheus: From 4b43a7797a63a3701417b2e6b7e41dc6bece6f2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Thu, 5 Dec 2019 19:23:13 +0100 Subject: [PATCH 088/257] docs/service-discovery.md: clean up (#1844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit cleans up some of the language in the service discovery talk for improved legibility. Signed-off-by: Lucas Servén Marín --- docs/service-discovery.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/service-discovery.md b/docs/service-discovery.md index d9f15074a6..23bd3ca899 100644 --- a/docs/service-discovery.md +++ b/docs/service-discovery.md @@ -7,15 +7,15 @@ slug: /service-discovery.md # Service Discovery -Service discovery has a vital place in Thanos components. It allows Thanos to discover different set API targets required to perform certain operations. +Service Discovery (SD) is a vital part of several Thanos components. It allows Thanos to discover API targets required to perform certain operations. -Currently places that uses Thanos SD: +SD is currently used in the following places within Thanos: * `Thanos Query` needs to know about [StoreAPI](https://github.com/thanos-io/thanos/blob/d3fb337da94d11c78151504b1fccb1d7e036f394/pkg/store/storepb/rpc.proto#L14) servers in order to query metrics from them. * `Thanos Rule` needs to know about `QueryAPI` servers in order to evaluate recording and alerting rules. -* (Only static option with DNS discovery): `Thanos Rule` needs to know about `Alertmanagers` HA replicas in order to send alerts. +* `Thanos Rule` needs to know about `Alertmanagers` HA replicas in order to send alerts; only static option with DNS discovery. -Currently there are several ways to configure this and they are described below in details: +There are currently several ways to configure SD, described below in more detail: * Static Flags * File SD @@ -39,10 +39,10 @@ The repeatable flag `--alertmanager.url=` can be used to specify a File Service Discovery is another mechanism for configuring components. With File SD, a list of files can be watched for updates, and the new configuration will be dynamically loaded when a change occurs. -The list of files to watch is passed to a component via a flag shown in the component specific sections below. +The list of files to watch is passed to a component via a flag shown in the component-specific sections below. The format of the configuration file is the same as the one used in [Prometheus' File SD](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config). -Both YAML and JSON files can be used. The format of the files is this: +Both YAML and JSON files can be used. The format of the files is as follows: * JSON: @@ -60,7 +60,7 @@ Both YAML and JSON files can be used. The format of the files is this: - targets: ['localhost:9090', 'example.org:443'] ``` -As a fallback, the file contents are periodically re-read at an interval that can be set using a flag specific for the component and shown below. +As a fallback, the file contents are periodically re-read at an interval that can be set using a flag specific to the component as shown below. The default value for all File SD re-read intervals is 5 minutes. ### Thanos Query @@ -90,30 +90,30 @@ An example using this lookup with a static flag: --store=dns+stores.thanos.mycompany.org:9090 ``` -* `dnssrv+` - the domain name after this prefix will be looked up as a SRV query, and then each SRV record will be looked up as an A/AAAA query. You do not need to specify a port as the one from the query results will be used. An example: +* `dnssrv+` - the domain name after this prefix will be looked up as a SRV query, and then each SRV record will be looked up as an A/AAAA query. You do not need to specify a port as the one from the query results will be used. For example: ``` --store=dnssrv+_thanosstores._tcp.mycompany.org ``` -It is also work in kubernetes cluster. An example: +DNS SRV record discovery also work well within Kubernetes. Consider the following example: ``` --store=dnssrv+_grpc._tcp.thanos-store.monitoring.svc ``` -It means that in kubernetes cluster, there is a service named "thanos-store" in `monitoring` namespace with a port defined named "grpc". +This configuration will instruct Thanos to discover all endpoints within the `thanos-store` service in the `monitoring` namespace and use the declared port named `grpc`. -* `dnssrvnoa+` - the domain name after this prefix will be looked up as a SRV query, with no A/AAAA lookup made after that. Similar to the `dnssrv+` case, you do not need to specify a port. An example: +* `dnssrvnoa+` - the domain name after this prefix will be looked up as a SRV query, with no A/AAAA lookup made after that. Similar to the `dnssrv+` case, you do not need to specify a port. For example: ``` --store=dnssrvnoa+_thanosstores._tcp.mycompany.org ``` -The default interval between DNS lookups is 30s. You can change it using the `store.sd-dns-interval` flag for `StoreAPI` +The default interval between DNS lookups is 30s. This interval can be changed using the `store.sd-dns-interval` flag for `StoreAPI` configuration in `Thanos Query`, or `query.sd-dns-interval` for `QueryAPI` configuration in `Thanos Rule`. ## Other -Currently, there are no plans of adding other Service Discovery mechanisms like Consul SD, kube SD, etc. However, we welcome -people implementing their preferred Service Discovery by writing the results to File SD which will propagate them to the different Thanos components. +Currently, there are no plans of adding other Service Discovery mechanisms like Consul SD, Kubernetes SD, etc. However, we welcome +people implementing their preferred Service Discovery by writing the results to File SD, which can be consumed by the different Thanos components. From fc576c5f094c8b84ed782be7b5be7bec1d0bc3ae Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 6 Dec 2019 18:50:37 +0000 Subject: [PATCH 089/257] Updated process of developing Katacoda tutorials. (#1803) Signed-off-by: Bartek Plotka --- tutorials/katacoda/README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tutorials/katacoda/README.md b/tutorials/katacoda/README.md index 8c124112ef..fbd883ff8b 100644 --- a/tutorials/katacoda/README.md +++ b/tutorials/katacoda/README.md @@ -2,8 +2,18 @@ Definitions of our courses for Katacoda. -## Development. +## Development -Find docs [here](https://katacoda.com/docs) +Process of adding / editing tutorials: -Thanos repo is hooked to [this](https://katacoda.com/bwplotka) Katacoda account. \ No newline at end of file +1. Sign up to [Katacoda website](https://katacoda.com). +1. Link your fork of Thanos to your account, see [this](https://katacoda.com/docs/configure-git). +1. Add / edit tutorials. +1. Push to `master` of your fork. + * You should see the updated preview of the tutorials on https://katacoda.com/. +1. Make a PR to Thanos repo. +1. Once PR is merged the official profile https://katacoda.com/thanos will be updated. + +Find more docs [here](https://katacoda.com/docs) + +Thanos repo is hooked to [this](https://katacoda.com/thanos) Katacoda account. \ No newline at end of file From d76f01b05e28800344b380315ce1c66d1f5631d8 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Sat, 7 Dec 2019 02:22:21 +0100 Subject: [PATCH 090/257] docs/storage: fix a typo and yaml formatting (#1847) Signed-off-by: Jakob Ackermann --- docs/storage.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/storage.md b/docs/storage.md index 2dbf1245d7..338b558f2c 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -18,7 +18,7 @@ We recommend the latter as it gives an explicit static view of configuration for Don't be afraid of multiline flags! -In Kubernetes it is as easy as (on Thanos sidecar example):: +In Kubernetes it is as easy as (on Thanos sidecar example): ```yaml - args: @@ -326,8 +326,8 @@ To use Tencent COS as storage store, you should apply a Tencent Account to creat To configure Tencent Account to use COS as storage store you need to set these parameters in yaml format stored in a file: -[embedmd]:# (flags/config_bucket_cos.txt $) -```$ +[embedmd]:# (flags/config_bucket_cos.txt yaml) +```yaml type: COS config: bucket: "" @@ -344,8 +344,8 @@ In order to use AliYun OSS object storage, you should first create a bucket with To use AliYun OSS object storage, please specify following yaml configuration file in `objstore.config*` flag. -[embedmd]:# (flags/config_bucket_aliyunoss.txt $) -```$ +[embedmd]:# (flags/config_bucket_aliyunoss.txt yaml) +```yaml type: ALIYUNOSS config: endpoint: "" From 15b807e179b4f1949396255418870c47e217284e Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Mon, 9 Dec 2019 18:44:40 +0800 Subject: [PATCH 091/257] comment url scheme (#1835) Signed-off-by: Xiang Dai <764524258@qq.com> --- cmd/thanos/rule.go | 2 +- docs/components/rule.md | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index b3f3db72ef..115be52ff8 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -81,7 +81,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { tsdbRetention := modelDuration(cmd.Flag("tsdb.retention", "Block retention time on local disk."). Default("48h")) - alertmgrs := cmd.Flag("alertmanagers.url", "Alertmanager replica URLs to push firing alerts. Ruler claims success if push to at least one alertmanager from discovered succeeds. The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Alertmanager IPs through respective DNS lookups. The port defaults to 9093 or the SRV record's value. The URL path is used as a prefix for the regular Alertmanager API path."). + alertmgrs := cmd.Flag("alertmanagers.url", "Alertmanager replica URLs to push firing alerts. Ruler claims success if push to at least one alertmanager from discovered succeeds. The scheme should not be empty e.g `http` might be used. The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Alertmanager IPs through respective DNS lookups. The port defaults to 9093 or the SRV record's value. The URL path is used as a prefix for the regular Alertmanager API path."). Strings() alertmgrsTimeout := cmd.Flag("alertmanagers.send-timeout", "Timeout for sending alerts to alertmanager").Default("10s").Duration() diff --git a/docs/components/rule.md b/docs/components/rule.md index 359c0572a4..9b7e92b2d1 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -201,12 +201,13 @@ Flags: Alertmanager replica URLs to push firing alerts. Ruler claims success if push to at least one alertmanager from discovered - succeeds. The scheme may be prefixed with - 'dns+' or 'dnssrv+' to detect Alertmanager IPs - through respective DNS lookups. The port - defaults to 9093 or the SRV record's value. The - URL path is used as a prefix for the regular - Alertmanager API path. + succeeds. The scheme should not be empty e.g + `http` might be used. The scheme may be + prefixed with 'dns+' or 'dnssrv+' to detect + Alertmanager IPs through respective DNS + lookups. The port defaults to 9093 or the SRV + record's value. The URL path is used as a + prefix for the regular Alertmanager API path. --alertmanagers.send-timeout=10s Timeout for sending alerts to alertmanager --alert.query-url=ALERT.QUERY-URL From f151d47b9a1c6c17d35a4524ca846f628c0418e6 Mon Sep 17 00:00:00 2001 From: Ziqi Zhao Date: Mon, 9 Dec 2019 20:43:47 +0800 Subject: [PATCH 092/257] skip the error of unmarshal for index-cache file (#1837) * skip the error of unmarshal for index-cache file Signed-off-by: fatsheep9146 * clean up index-cache file when crashed Signed-off-by: fatsheep9146 --- pkg/block/index.go | 7 ++++++- pkg/store/bucket.go | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/block/index.go b/pkg/block/index.go index 4789bea794..4d45cc5565 100644 --- a/pkg/block/index.go +++ b/pkg/block/index.go @@ -20,6 +20,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/oklog/ulid" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunks" @@ -32,6 +33,10 @@ const ( IndexCacheVersion1 = iota + 1 ) +var ( + IndexCacheUnmarshalError = errors.New("unmarshal index cache") +) + type postingsRange struct { Name, Value string Start, End int64 @@ -194,7 +199,7 @@ func ReadIndexCache(logger log.Logger, fn string) ( } if err = json.Unmarshal(bytes, &v); err != nil { - return 0, nil, nil, nil, errors.Wrap(err, "unmarshal index cache") + return 0, nil, nil, nil, errors.Wrap(IndexCacheUnmarshalError, err.Error()) } strs := map[string]string{} diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index c45c8c973a..d82690d7c3 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -1255,7 +1255,7 @@ func (b *bucketBlock) loadIndexCacheFile(ctx context.Context) (err error) { if err = b.loadIndexCacheFileFromFile(ctx, cachefn); err == nil { return nil } - if !os.IsNotExist(errors.Cause(err)) { + if !os.IsNotExist(errors.Cause(err)) && errors.Cause(err) != block.IndexCacheUnmarshalError { return errors.Wrap(err, "read index cache") } @@ -1264,7 +1264,7 @@ func (b *bucketBlock) loadIndexCacheFile(ctx context.Context) (err error) { return b.loadIndexCacheFileFromFile(ctx, cachefn) } - if !b.bucket.IsObjNotFoundErr(errors.Cause(err)) { + if !b.bucket.IsObjNotFoundErr(errors.Cause(err)) && errors.Cause(err) != block.IndexCacheUnmarshalError { return errors.Wrap(err, "download index cache file") } From 7426f1f78bd188bdd4a01446b06701c51035788e Mon Sep 17 00:00:00 2001 From: Rob Best Date: Mon, 9 Dec 2019 17:00:28 +0000 Subject: [PATCH 093/257] bump minio-go dep to v6.0.44 (#1852) Signed-off-by: Rob Best --- CHANGELOG.md | 2 ++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3af82e0be0..c655527ee0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.) ## Unreleased +### Added +- [#1852](https://github.com/thanos-io/thanos/pull/1852) Add support for `AWS_CONTAINER_CREDENTIALS_FULL_URI` by upgrading to minio-go v6.0.44 ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 diff --git a/go.mod b/go.mod index 141106d1af..171dd7188a 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe // indirect github.com/mattn/go-runewidth v0.0.6 // indirect github.com/miekg/dns v1.1.22 - github.com/minio/minio-go/v6 v6.0.41 + github.com/minio/minio-go/v6 v6.0.44 github.com/mozillazg/go-cos v0.13.0 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/oklog/run v1.0.0 diff --git a/go.sum b/go.sum index 9964e1ec60..42cbdc4313 100644 --- a/go.sum +++ b/go.sum @@ -350,8 +350,8 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc= github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/minio/minio-go/v6 v6.0.41 h1:ybV6itOYLsjCpp4+QjE4gokic/xhJU7D7wRzxqPHTio= -github.com/minio/minio-go/v6 v6.0.41/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/minio-go/v6 v6.0.44 h1:CVwVXw+uCOcyMi7GvcOhxE8WgV+Xj8Vkf2jItDf/EGI= +github.com/minio/minio-go/v6 v6.0.44/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= From 352cc30545e5bca6e2bde1ae67fd9785cf49ef2f Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 9 Dec 2019 18:08:27 -0600 Subject: [PATCH 094/257] receive: close DBReadOnly after flushing (#1856) Signed-off-by: Brett Jones --- CHANGELOG.md | 4 ++++ pkg/receive/tsdb.go | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c655527ee0..81637a35c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.) ## Unreleased +### Fixed + +- [#1856](https://github.com/thanos-io/thanos/pull/1856) Receive: close DBReadOnly after flushing to fix a memory leak. + ### Added - [#1852](https://github.com/thanos-io/thanos/pull/1852) Add support for `AWS_CONTAINER_CREDENTIALS_FULL_URI` by upgrading to minio-go v6.0.44 diff --git a/pkg/receive/tsdb.go b/pkg/receive/tsdb.go index da463fe874..8a081f0601 100644 --- a/pkg/receive/tsdb.go +++ b/pkg/receive/tsdb.go @@ -92,6 +92,7 @@ func (f *FlushableStorage) Flush() error { if err != nil { return errors.Wrap(err, "opening read-only DB") } + defer ro.Close() if err := ro.FlushWAL(f.path); err != nil { return errors.Wrap(err, "flushing WAL") } From 5ed75d0c305781e7f439fd49131aa4ee0ba66107 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 10 Dec 2019 00:43:00 +0000 Subject: [PATCH 095/257] Added comment to non obvious downsampling aggregation. (#1853) Signed-off-by: Bartek Plotka --- pkg/compact/downsample/downsample.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/compact/downsample/downsample.go b/pkg/compact/downsample/downsample.go index f51ce7cb02..e64e3ceb02 100644 --- a/pkg/compact/downsample/downsample.go +++ b/pkg/compact/downsample/downsample.go @@ -474,6 +474,8 @@ func downsampleAggrBatch(chks []*AggrChunk, buf *[]sample, resolution int64) (ch return nil } if err := do(AggrCount, func(a *aggregator) float64 { + // To get correct count of elements from already downsampled count chunk + // we have to sum those values. return a.sum }); err != nil { return chk, err From e94478c81c2615adde63ed56930ee0114b7b3c5a Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 9 Dec 2019 19:51:47 -0500 Subject: [PATCH 096/257] Update rule UI (#1854) * update rule ui Signed-off-by: yeya24 * typo fix Signed-off-by: yeya24 * update changelog Signed-off-by: yeya24 --- .circleci/config.yml | 2 +- CHANGELOG.md | 1 + Makefile | 2 +- pkg/ui/bindata.go | 144 ++++++++++++++++---------------- pkg/ui/rule.go | 49 ++++++----- pkg/ui/static/js/alerts.js | 83 +++++++++++++------ pkg/ui/templates/alerts.html | 155 ++++++++++++++++++++--------------- 7 files changed, 250 insertions(+), 186 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 29845cfcfb..c96ebc1da7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: working_directory: /go/src/github.com/thanos-io/thanos environment: GO111MODULE: 'on' - # Run garbage collection more aggresively to avoid getting OOMed during the lint phase (4GB limit). + # Run garbage collection more aggressively to avoid getting OOMed during the lint phase (4GB limit). GOGC: "20" # By default Go uses GOMAXPROCS but a Circle CI executor has many # cores (> 30) while the CPU and RAM resources are throttled. If we diff --git a/CHANGELOG.md b/CHANGELOG.md index 81637a35c7..a05801398a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Added - [#1852](https://github.com/thanos-io/thanos/pull/1852) Add support for `AWS_CONTAINER_CREDENTIALS_FULL_URI` by upgrading to minio-go v6.0.44 +- [#1854](https://github.com/thanos-io/thanos/pull/1854) Update Rule UI to support alerts count displaying and filtering. ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 diff --git a/Makefile b/Makefile index b9de15fc2b..29c7995b25 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ endef .PHONY: all all: format build -# assets repacks all statis assets into go file for easier deploy. +# assets repacks all static assets into go file for easier deploy. .PHONY: assets assets: $(GOBINDATA) @echo ">> deleting asset file" diff --git a/pkg/ui/bindata.go b/pkg/ui/bindata.go index c189484a7f..27db965494 100644 --- a/pkg/ui/bindata.go +++ b/pkg/ui/bindata.go @@ -160,12 +160,12 @@ func pkgUiTemplates_baseHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/_base.html", size: 1478, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/_base.html", size: 1478, mode: os.FileMode(420), modTime: time.Unix(1575924708, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _pkgUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\x5f\x6f\xdb\x36\x10\x7f\xcf\xa7\x38\xa8\x7d\xd8\x80\xc9\x42\x90\xf6\x61\x36\xad\x21\xe8\xcb\x1e\xd2\x62\x48\xd2\xbc\x06\x14\x79\xb6\x58\x33\xa4\x46\xd2\x8e\x3d\x4e\xdf\x7d\x20\x29\x39\xb2\x2d\xad\x1d\x30\x18\x10\xc8\xbb\xe3\xef\x7e\xf7\x87\x47\x7b\xcf\x71\x25\x14\x42\x56\x23\xe5\x59\xdb\x5e\x01\x10\x29\xd4\x06\xdc\xa1\xc1\x65\xe6\x70\xef\x0a\x66\x6d\x06\x06\xe5\x32\xb3\xee\x20\xd1\xd6\x88\x2e\x83\xda\xe0\x6a\x99\x79\x0f\x0d\x75\xf5\x1f\x06\x57\x62\x0f\x6d\x5b\x58\x47\x9d\x60\xe1\x4c\x41\x25\x1a\x67\x67\xcc\xda\xdf\x76\x4b\xef\xa1\xda\x0a\xc9\x9f\xd0\x58\xa1\x15\xb4\x6d\x56\x06\x67\x96\x19\xd1\x38\xb0\x86\x4d\x83\x7d\x3b\x62\x7d\x9b\x82\x22\x45\x02\x2a\xaf\xbc\x47\xc5\xdb\xf6\xea\xea\x2d\x36\xa6\x95\x43\xe5\x42\x78\x84\x8b\x1d\x30\x49\xad\x5d\x46\x31\x15\x0a\x4d\xbe\x92\x5b\xc1\x13\x9f\xfa\xba\xbc\x8d\xbe\x48\x51\x5f\x47\xc9\xe0\x84\xad\xf5\x6b\x4e\x95\xd2\x81\x97\x56\x36\x1e\x01\x00\x22\x7a\x8b\xb5\x3c\x34\xb5\x60\x5a\xc1\x71\x95\x6f\x15\xab\x91\x6d\x90\x07\x9a\xa2\x3b\x02\x40\xaa\xad\x73\x5a\x75\x99\x4e\x9b\x6c\xd2\x13\x38\xe1\x24\x26\x05\x9c\x50\x78\x38\x93\x90\x22\x61\x45\xf2\x05\x17\xbb\xb8\x70\xb4\x92\xd8\xa3\xa7\x4d\xfc\xe6\x95\x36\x1c\x0d\xf2\x6e\xcb\xb4\x94\xb4\xb1\xc8\xbb\xd8\x88\xab\x34\x3f\xa4\xb5\xf7\xef\x63\x1d\x1e\x1c\x75\xf8\xa8\xef\xf5\xeb\xa7\x80\x07\xf3\x25\xcc\x6e\x47\x14\xb1\x9d\xc2\x31\x43\xd5\x1a\x3b\x1b\xa1\xd6\xf7\x5b\x89\xbd\x32\xa1\x32\x27\x76\x98\xf2\x9e\xd0\x06\x82\xa3\x21\x71\xa6\x0f\x20\xd2\x80\xf8\xcd\xbd\x17\x8a\xe3\x1e\xc6\xb9\xcd\xa2\xa0\x6d\x93\xf1\x73\x68\x73\x34\xd9\xb1\x08\x40\x1c\x2f\xdf\xca\x17\xeb\xc5\x6a\xdc\x19\xad\x72\xae\x5f\x55\x2a\x19\x90\xaa\xf4\x7e\xf6\x85\xbe\x60\xdb\x92\xa2\x2a\xe1\x27\xef\x25\x2a\x38\x61\x1e\x9c\xc4\xed\xcf\xa4\x70\xbc\x77\x41\x0a\x67\xca\x89\x08\x9e\x39\x3a\x2a\xa4\x3d\xe3\x73\xdc\xa4\xee\x1b\xee\x01\x48\x63\x10\xe2\x45\x5c\x66\x5c\xd8\x46\xd2\xc3\xbc\x92\x9a\x6d\x16\xd0\x50\xce\x85\x5a\xcf\x7f\x9d\x7d\x6c\xf6\x0b\x58\x69\xe5\x72\x2b\xfe\xc2\xf9\xf5\x4d\xd8\x33\x2d\xb5\x99\xbf\xbb\xb9\xb9\x59\xc0\xab\x36\x3c\xaf\x0c\xd2\xcd\x3c\x7e\x73\x2a\xe5\x02\x2a\xca\x36\x6b\xa3\xb7\x8a\xe7\x9d\xf1\xea\x63\xf8\x2d\x20\x75\xc9\xfc\xba\xd9\x83\xd5\x52\x70\x78\xc7\x18\xeb\xc5\xb9\xa1\x5c\x6c\xed\xfc\x43\xb3\x5f\x64\x50\x12\xa6\x39\x86\x7c\xfd\xfe\xf8\xf9\xee\x41\x89\xa6\x41\x07\x7f\x6e\xd1\x1c\xbe\xde\xdf\x85\xfc\x45\x3d\x29\x1a\x83\x27\xa1\x16\x67\xb1\x7a\x2f\x56\xe7\x19\x1e\xda\xff\x68\x4b\xd7\x7a\x87\xa6\x5b\xdb\x97\xae\x11\x50\xe2\x0b\x2a\x67\x9f\xa3\x3c\x3b\xcb\xf1\x5b\x9d\xce\x34\x41\x57\x97\x77\xb4\x42\x69\x49\xe1\xea\x31\x6d\xec\xb8\x29\x65\xea\x6c\x78\x10\x8a\x4d\xda\x3c\x51\xb9\x1d\x51\x0e\x3b\xa9\xcf\x50\xba\x59\xd3\x49\x8a\xb1\x5c\xfa\xe0\xe7\xa2\x01\x96\x0c\xc1\xfd\x02\xef\x77\x81\x45\xbc\x8d\x29\xdc\xd9\x67\xda\x9c\x61\x77\x70\xb6\xa1\xaa\xcf\x57\x45\xf9\x1a\x21\x7e\xf3\xc6\x88\x17\x6a\x0e\x59\xe9\x7d\x42\x6d\xdb\x30\xe4\x13\x72\xdb\x66\xa4\x08\x27\xc7\xa8\xa4\x11\x7e\xe6\xa6\xb8\xa4\x1d\x6f\xef\xd0\xfd\xe9\x60\xe8\x6e\x3f\xfc\x0d\xc3\xd9\x90\x06\x43\xdb\x42\x78\x5e\xf0\x59\x28\x2e\x18\x75\xda\x40\x78\xed\xf2\x6d\xd3\xa0\x61\xd4\x62\xa0\xdd\x4f\x8f\x8e\xe9\x14\x05\xef\xfb\x89\xe5\x66\x5f\x1f\x3f\x05\xfb\x49\xc3\xa7\x14\xfc\xa5\xc5\x58\x79\x41\xac\x60\x76\xfb\x36\xdb\x47\x6a\x10\x7a\xf5\x6c\x1c\x28\xad\x30\x3b\x1d\x34\x23\x8f\xd6\x00\xa1\x0e\xb3\x21\x44\xb8\xcc\x3e\x64\xe5\xed\xf0\x2d\xf9\x7e\x13\xfe\x1f\x04\xf8\x09\x81\x8b\x86\x20\x5c\xfe\xa7\x86\xfd\xf7\x8c\xf5\x98\x6e\xd0\x97\xa4\xe0\xee\xd2\x45\xb0\x0a\x45\xeb\x3b\x96\x14\x7c\xf4\xe6\x8c\xb5\x6b\x1c\x68\x17\xb4\x7f\xac\xec\x97\x78\x97\x32\x52\xc4\xc9\x75\x3a\x30\x4f\x8d\xc6\x9f\x21\xef\x51\x5a\x1c\xbe\xaa\x93\xaf\x0f\xc0\x17\x9d\xae\x8e\x50\x6b\x30\xe1\xd9\x86\xf4\xa7\x8a\x7f\xdf\xc9\x91\x0a\x29\x8e\xff\x21\x8e\xa4\xbb\x61\xdf\x9b\xfd\x13\x00\x00\xff\xff\x4e\x7c\x2e\x8d\x88\x0a\x00\x00") +var _pkgUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x58\x5b\x6f\xdb\x36\x14\x7e\xcf\xaf\x38\x50\xfb\xb0\x02\x93\x85\x2c\xed\xc3\x1c\xd9\x43\x50\xa0\xdb\x80\xb6\x2b\x9a\xac\xaf\x01\x45\x1e\x5b\x6c\x18\x92\x20\x69\xc7\x99\xa6\xff\x3e\x90\x94\x7c\x51\xe5\x9b\x0a\x6c\x73\x00\x41\x22\x79\x2e\xdf\xb9\x33\x55\xc5\x70\xc6\x25\x42\x52\x22\x61\x49\x5d\x5f\x00\x00\xe4\x82\xcb\x07\x70\xcf\x1a\x27\x89\xc3\x95\xcb\xa8\xb5\x09\x18\x14\x93\xc4\xba\x67\x81\xb6\x44\x74\x09\x94\x06\x67\x93\xa4\xaa\x40\x13\x57\x7e\x32\x38\xe3\x2b\xa8\xeb\xcc\x3a\xe2\x38\xf5\x34\x19\x11\x68\x9c\x1d\x51\x6b\x7f\x59\x4e\xaa\x0a\x8a\x05\x17\xec\x0b\x1a\xcb\x95\x84\xba\x4e\xa6\x51\x9c\xa5\x86\x6b\x07\xd6\xd0\xfd\xec\xbe\xae\xb9\x7d\xdd\xc7\x2c\xcf\x22\xa3\xe9\x45\x55\xa1\x64\x75\x7d\x71\xb1\xc1\x47\x95\x74\x28\xdd\x1a\x22\xe3\x4b\xa0\x82\x58\x3b\x09\x5b\x84\x4b\x34\xe9\x4c\x2c\x38\x6b\xb4\x0a\xa7\xca\xcb\xe9\x4d\x90\x9a\x67\xe5\xe5\xd6\xba\xa7\xe6\x6c\x92\x04\x95\xde\x71\xe1\xd0\xd8\xa4\xe5\x57\x38\x99\xce\x8d\x5a\x68\x58\xbf\xa5\x4e\xcd\xe7\x02\x13\x60\xc4\x91\xe6\x63\x92\x14\x0b\xe7\x94\xb4\x5b\x02\xa3\xf5\x49\x81\x62\x8b\x59\x60\xa3\x0d\x7f\x24\xe6\x19\x08\x75\x7c\x89\x1d\x92\x40\xc6\xa5\x5e\xb8\xc6\x6b\xb4\x44\xfa\x50\xa8\x55\x02\x92\x3c\x62\x57\x4f\xaf\x39\x97\x91\x53\x84\x97\x00\x59\x38\x45\xd5\xa3\x16\xe8\x70\x92\xa8\xd9\x2c\x99\xc2\xef\xcd\x19\xf8\xa1\xaa\x60\xf4\x56\x2d\xa4\xb3\xa3\xf5\x62\x5d\xbf\xda\xd5\x3b\x0b\x8a\xff\x17\x60\x34\x4a\xc6\xe5\xfc\x10\x96\x4f\xf1\xc8\x0e\x94\x76\xed\x7f\x84\x64\xc6\xcd\x11\x20\xef\xc2\x89\x1d\x1c\xcd\xd2\x69\x30\xb2\xc2\x6c\x05\x72\xc6\xf8\xb2\x13\xd7\x0d\x42\x5b\xaa\xa7\x94\x48\xa9\x7c\xfe\xf5\x44\x29\x6f\x0f\xce\xc5\xb3\x2e\x39\x55\x12\xd6\x6f\xe9\x42\x06\xa8\xc8\x7c\x56\xf2\x0e\x65\x0c\xfb\xc6\x24\xf1\x23\xd9\x2b\x15\x1c\x77\x3e\x57\xfc\x06\xec\xa8\x73\xdb\x59\xc9\xb3\xc8\x6b\x3f\x3a\x47\x0a\x81\x9b\xbc\xb5\x77\xfe\x7b\x2d\x3a\xee\x86\x67\x5a\x28\xc3\xd0\x20\x6b\x3e\xa9\x12\x82\x68\x8b\xac\x6b\x04\x57\x28\xf6\xbc\xbb\x56\x55\x2f\x03\xf7\x5b\x47\x1c\xde\xa9\xcf\xea\xe9\xad\xe7\x0f\xe3\x09\x8c\x6e\x7a\x36\x9a\x82\xb4\x21\x37\x44\xce\x11\x46\xbf\xfa\xba\xd1\xdd\x8d\x42\xcd\xb7\x61\x16\x37\x18\x84\xfa\x3c\x49\x34\x61\x3e\xb2\xc7\xf0\x93\x5e\xf5\x04\xe5\x46\xd8\xe8\x0f\xc3\xe7\x5c\x12\xf1\x8e\x0b\xac\x6b\x98\xfa\xb5\x8f\xe4\x11\x7b\x04\x47\x93\x3a\xd6\x13\xe4\x59\x9f\x4e\x6b\x28\x01\x36\x97\xf3\xcf\x0b\x81\x7d\x88\x5a\xb3\x6d\x15\xa4\x68\xae\xad\x85\x7d\xfa\x38\xd3\xfa\x2f\x58\x1d\xc2\x33\xad\x2a\x2e\x19\xae\xa0\xdf\x15\xa3\xb0\x50\xd7\xf1\xf0\xbd\x6f\x7d\x68\x0e\x98\x29\x77\x6c\xba\x89\xf7\x10\xe0\xb4\xc4\xa5\x51\x32\x65\xea\x49\xc6\x18\x87\xbc\x98\xae\x6d\x97\x67\xc5\xd4\xa7\xa8\x40\x09\x3b\xb8\xbc\xd0\xf0\xf9\xaa\xdf\x92\xfb\xad\xd9\x87\xf6\x9e\xa1\x23\x5c\x74\x73\xb3\xab\xfb\xde\x4d\x68\x72\xfe\xf0\x89\x70\x4a\x1b\x6c\x83\x8b\x71\xab\x05\x79\x1e\x17\x42\xd1\x87\x6b\x68\x63\xed\xe7\xd1\x1b\xbd\xba\x86\x99\x92\x2e\xb5\xfc\x2f\x1c\x5f\x5e\xf9\x6f\xaa\x84\x32\xe3\x17\x57\x57\x57\xd7\xf0\xa4\x0c\x4b\x0b\x83\xe4\x61\x1c\x9e\x29\x11\xe2\x1a\x0a\x42\x1f\x7c\x97\x94\x2c\x6d\x0e\xcf\xde\xf8\xbf\x6b\x88\x69\x38\xbe\xd4\x2b\xb0\x4a\x70\x06\x2f\x28\xa5\xed\x72\x6a\x08\xe3\x0b\x3b\x7e\xad\x57\xd7\x09\x4c\x73\xaa\x18\x7a\x0f\xfc\x76\xf7\xe1\xfd\xad\xe4\x5a\xa3\xdb\x1a\x25\xbc\x4f\xc2\x89\x3c\xd3\x06\x8f\x98\x24\x3b\x6a\x93\xaa\xe2\xb3\xae\x67\x8f\x1b\x31\x96\x98\xe3\xf5\xa6\x54\x4b\x34\xcd\xbb\x7d\x6c\xc2\x14\x05\x3e\xa2\x74\xf6\x3e\xac\x1f\x70\xf9\xae\xc8\x75\xc4\x9c\x48\x11\xa9\xca\xe9\x7b\xdf\x3b\x6c\x9e\xb9\xf2\x3c\xba\x90\x5b\xe7\x93\xc5\x5c\x87\x5b\x2e\xe9\x00\xea\x2f\x44\x2c\xce\x20\xdb\x9f\x61\xdd\x5f\x5b\xc3\xce\x75\xf5\x46\xb9\x13\x05\x6d\x08\x8e\xe4\xeb\x41\x2d\x43\xc3\xff\x11\x5e\x2e\xbd\x3d\x42\x0d\x8d\x6e\x1c\x7d\x20\xfa\x0c\xad\x77\x14\xb2\x9a\xc8\xf5\xd8\x43\xd8\x1c\x21\x3c\xdb\xd1\x27\x99\x56\x55\x94\x5b\xd7\x7e\x7c\x8f\xb2\xeb\x3a\xc9\x33\x4f\x39\x04\x4c\x1c\xde\xcf\x52\x72\x6f\x31\xdd\x4b\xe1\x6b\xfa\x36\xb4\xdd\xf6\xd1\xf4\x08\xf8\x1b\xb6\x3b\x48\x6c\x1f\x75\x0d\xfe\x52\x82\xf7\x5c\x32\x4e\x89\x53\x06\xfc\x2d\x29\x5d\x68\x8d\x86\x12\x8b\xde\x24\x6d\x8f\x69\xac\x30\x4c\xc1\xaa\x6a\x7b\xa0\x1b\xfd\x79\xf7\xd6\x73\x1b\xc8\xe6\x4b\x74\xca\x79\xf4\xa7\xa7\x09\x04\xbf\x01\x9f\xc1\xe8\x66\x33\x91\x0d\x88\x3a\x5f\xaf\x3a\x6d\x46\x2a\xb9\x19\xd2\x62\x29\xdc\x3f\x96\x9e\x26\xa3\xf4\x5d\xc9\xfb\x65\x92\xbc\x4e\xa6\x37\xdb\x33\xe4\x39\x95\x07\xce\xb6\xd1\xbf\x07\x91\xed\x40\x1c\x96\xf9\x4c\x0c\x23\x84\xc3\x25\xe9\xfb\x22\xe4\x5b\x2d\xdd\x56\x05\xca\x33\xe6\x86\x2b\x1d\xf9\xf9\x84\x69\xab\x58\x9e\xb1\x01\xf5\xb8\xfd\x0d\x29\x65\x6b\x3d\xb2\x21\xe6\x1f\x50\x20\xce\x4c\xf2\xd3\x11\x9d\x7a\x36\xcf\xc2\x44\x73\x6c\xe0\x3a\xcc\xec\xdc\x71\xba\x8f\x5f\x55\xa1\xb0\x7d\x37\x9f\x43\x57\xae\xfd\x6a\x7f\x54\xb1\x79\xf8\xbb\xb9\xf1\x57\x1f\x88\xff\x8c\x62\xdf\x75\xb3\xea\x53\x3c\xcf\x3a\x17\xd1\x1d\x93\x36\xe3\x6c\x4b\xf8\x4f\x00\x00\x00\xff\xff\xe3\xad\x3c\xbb\xf4\x13\x00\x00") func pkgUiTemplatesAlertsHtmlBytes() ([]byte, error) { return bindataRead( @@ -180,7 +180,7 @@ func pkgUiTemplatesAlertsHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 2696, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 5108, mode: os.FileMode(420), modTime: time.Unix(1575922733, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -200,7 +200,7 @@ func pkgUiTemplatesBucketHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket.html", size: 1281, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket.html", size: 1281, mode: os.FileMode(420), modTime: time.Unix(1564066864, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -220,7 +220,7 @@ func pkgUiTemplatesBucket_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 785, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 785, mode: os.FileMode(420), modTime: time.Unix(1564066864, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -240,7 +240,7 @@ func pkgUiTemplatesGraphHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/graph.html", size: 2298, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/graph.html", size: 2298, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -260,7 +260,7 @@ func pkgUiTemplatesQuery_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1364, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1364, mode: os.FileMode(420), modTime: time.Unix(1564066864, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -280,7 +280,7 @@ func pkgUiTemplatesRule_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 961, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 961, mode: os.FileMode(420), modTime: time.Unix(1564066864, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -300,7 +300,7 @@ func pkgUiTemplatesRulesHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rules.html", size: 1944, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rules.html", size: 1944, mode: os.FileMode(420), modTime: time.Unix(1575919007, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -320,12 +320,12 @@ func pkgUiTemplatesStatusHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/status.html", size: 1272, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/status.html", size: 1272, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _pkgUiTemplatesStoresHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x56\x51\x6b\xdb\x30\x10\x7e\xcf\xaf\x38\x4c\x5f\x13\x43\x5f\x06\xc3\xc9\x18\xa3\xb0\x87\xb6\x0c\xd2\xf5\x75\x28\xd6\x25\x16\x55\x24\xa3\x3b\xb7\x09\x42\xff\x7d\xc8\xb1\x13\xbb\x76\xb2\x64\x2f\x06\x9d\xee\xf4\x7d\xd6\xf7\xdd\xd9\xde\x4b\x5c\x2b\x83\x90\x14\x28\x64\x12\xc2\x24\xd3\xca\xbc\x01\xef\x4b\x9c\x27\x8c\x3b\x4e\x73\xa2\x04\x1c\xea\x79\x42\xbc\xd7\x48\x05\x22\x27\x50\x38\x5c\xcf\x13\xef\xa1\x14\x5c\xfc\x72\xb8\x56\x3b\x08\x21\x25\x16\xac\xf2\x58\x93\xba\x4a\x23\xcd\x72\xa2\x6f\xef\x73\xef\x61\x55\x29\x2d\x5f\xd1\x91\xb2\x06\x42\x48\x16\x13\xef\xd1\xc8\x10\x26\x93\x13\x89\xdc\x1a\x46\xc3\x35\x0f\xa9\xde\x21\xd7\x82\x68\x5e\x87\x85\x32\xe8\xa6\x6b\x5d\x29\x99\x2c\x26\x00\x00\xde\x3b\x61\x36\x08\x77\xc4\xd6\xe1\xcb\xbe\x44\xf8\x3a\x87\xd9\xd2\x56\x2e\x47\x0a\xa1\x49\x52\xeb\x4e\x46\x13\xcd\x8a\xfb\x85\xf7\xac\x58\x77\xcb\x67\x4b\x76\xca\x6c\x42\xc8\xd2\xe2\xbe\xc5\x40\x4d\xdd\xaa\xdf\xe6\xcd\xd8\x0f\x03\x31\xbf\x97\x56\xbf\x4a\x9d\xc5\x62\xa5\xb1\xa5\x7e\x58\xd4\xcf\xe9\xca\x3a\x89\x0e\x5b\xfe\x87\xe4\x78\xef\xdd\xb5\x3b\x2d\x9a\x84\xc5\x83\x91\xa5\x55\x86\xb3\x94\x8b\xe1\xee\x92\x05\x57\x34\xbe\xf7\xdd\x18\x5b\x99\x1c\x25\x3c\x8a\x15\xea\x25\xf2\x99\xc4\x27\x65\xe0\x45\x6d\xf1\xcc\xae\xd8\x5d\xd8\x7d\x14\xc4\xf0\x13\x85\xe6\x02\x7e\x14\x98\xbf\x5d\x48\x7b\x42\x22\xb1\xf9\x74\x50\x96\x76\xdf\x3a\xee\x7d\xba\x93\x95\x95\xfb\xd3\xba\xaf\x7b\xd4\x5c\x19\x89\x3b\xb8\x9b\x2d\x63\x80\x86\x72\x9f\xb9\x59\xb9\xf0\xfe\x90\x3b\x7b\x16\x5b\x8c\xba\xb3\x1c\x24\xb5\x4a\x46\x6b\x63\xd2\xdf\x86\xd6\x61\xc6\x72\x03\x3b\x8b\xef\xf9\xe0\x9c\x75\x1d\xf0\xe3\x71\x54\x0a\xd3\x1e\x28\x34\x3a\x86\xfa\x39\xa5\x2a\xcf\x91\x08\x6a\x90\x3f\xca\x48\x95\x0b\xb6\x0e\x62\x07\x4e\xab\xb2\x44\x97\x0b\x1a\x43\xaf\xca\x21\x48\x1a\x51\xc6\x88\x76\xbc\x7c\x15\x2b\x19\xef\xd9\xdd\x4e\x4a\xda\x0f\x73\x0b\xad\x63\xef\x9c\x72\x47\x84\x18\x96\xde\xd4\x6a\xa7\xa2\xbe\x9d\x4e\x3c\x1a\x5b\xe9\xb6\x55\xa2\xb5\x8e\xa2\x36\xb1\xb1\xeb\xfb\xec\xac\x8b\x9c\x87\x50\x35\xcc\x11\xf4\x00\x35\x86\x73\x3c\xb6\x2b\xd7\x4a\xc8\x0d\x42\xfd\x9c\x96\x4e\x6d\x85\xdb\x27\xd1\xd6\xf5\x79\x8d\xad\xe3\x9c\x6e\x02\xaf\x42\x57\x18\x42\x72\x4e\x0c\x38\x23\xc8\x11\x7b\x20\x0c\x0c\x1a\xf8\x5f\xe7\x64\xe9\x19\x05\xb2\xb4\x16\x6f\x71\x85\x15\xbc\x5f\x5b\xb7\x15\x1c\xa7\x12\xb1\xd8\x96\xad\x50\x4f\xca\xc4\xd8\x99\x5e\xbe\x50\x27\x76\x97\xeb\x48\x99\x1c\xbb\x3d\x5e\x8f\xba\x10\x40\x6c\xec\x95\x76\xed\x7c\x8b\x2e\x4e\x09\xb8\xbd\x27\x47\x7c\x7e\x40\xbc\x16\xee\xbf\x9b\xb3\xaf\xfd\x60\xc6\x8c\x8d\x5d\xc8\xad\x8e\x70\xf3\xe4\xcb\x08\xef\x67\x0b\x74\x98\xe3\x0e\x37\x8a\x38\xb6\xf1\x2d\xf8\x3d\xbe\x3d\xaf\xf5\xfc\x35\x64\xba\xea\x7e\x84\x3a\x7f\x1f\xdd\xdb\xff\x10\xce\x28\xb3\x49\x16\x63\x2c\xb3\x54\xaa\xf7\xfe\x0f\x41\x13\x6a\x97\x7f\x03\x00\x00\xff\xff\xb1\x86\x1f\x43\x6f\x09\x00\x00") +var _pkgUiTemplatesStoresHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x56\x41\x6b\xeb\x38\x10\xbe\xe7\x57\x0c\xa2\xd7\xc4\xd0\xcb\xc2\xe2\x78\x59\x96\xc2\x1e\xda\xf2\x20\x7d\xbd\x3e\x14\x6b\x12\x8b\x2a\x92\x91\xc6\x6d\x82\xd0\x7f\x7f\xc8\xb1\x13\xb9\x76\xd2\xf4\x5d\x0c\x1a\xcd\xcc\x37\x9e\xef\x9b\xb1\xbd\x17\xb8\x91\x1a\x81\x55\xc8\x05\x0b\x61\x96\x2b\xa9\xdf\x80\x0e\x35\x2e\x19\xe1\x9e\xb2\xd2\x39\x06\x16\xd5\x92\x39\x3a\x28\x74\x15\x22\x31\xa8\x2c\x6e\x96\xcc\x7b\xa8\x39\x55\x3f\x2c\x6e\xe4\x1e\x42\xc8\x1c\x71\x92\x65\x8c\xc9\x6c\xa3\xd0\x2d\x4a\xe7\xfe\x79\x5f\x7a\x0f\xeb\x46\x2a\xf1\x8a\xd6\x49\xa3\x21\x04\x56\xcc\xbc\x47\x2d\x42\x98\xcd\xce\x45\x94\x46\x13\x6a\x6a\xeb\x10\xf2\x1d\x4a\xc5\x9d\x5b\xb6\x66\x2e\x35\xda\xf9\x46\x35\x52\xb0\x62\x06\x00\xe0\xbd\xe5\x7a\x8b\x70\xe7\xc8\x58\x7c\x39\xd4\x08\x7f\x2f\x61\xb1\x32\x8d\x2d\xd1\x85\xd0\x39\xc9\x4d\xe2\xd1\x59\xf3\xea\xbe\xf0\x9e\x24\xa9\x34\x7c\xb1\x22\x2b\xf5\x36\x84\x3c\xab\xee\x7b\x0c\x54\x2e\x8d\xfa\xa9\xdf\xb4\xf9\xd0\x10\xfd\x07\x6e\xed\xab\xb4\x5e\xc4\xd7\x0a\xfb\xd2\x8f\x87\xf6\x39\x5f\x1b\x2b\xd0\x62\x5f\xff\xd1\x39\xf6\x3d\x3d\xdb\xf3\xa1\x73\x28\x1e\xb4\xa8\x8d\xd4\x94\x67\x54\x8d\x6f\x57\xc4\xa9\x71\xd3\x77\xff\x6a\x6d\x1a\x5d\xa2\x80\x47\xbe\x46\xb5\x42\xba\xe0\xf8\x24\x35\xbc\xc8\x1d\x5e\xb8\xe5\xfb\x2b\xb7\x8f\xdc\x11\xfc\x8f\x5c\x51\x05\xff\x55\x58\xbe\x5d\x71\x7b\x42\xe7\xf8\xf6\x53\xa2\x3c\x4b\xdf\x3a\xde\x7d\xea\xc9\xda\x88\xc3\xf9\x3c\xe4\x3d\x72\x2e\xb5\xc0\x3d\xdc\x2d\x56\xd1\xe0\xc6\x74\x5f\xe8\xac\x28\xbc\x3f\xfa\x2e\x9e\xf9\x0e\x23\xef\x24\x46\x4e\x3d\x93\x51\xda\xc8\x86\xd7\xd0\x2b\x4c\x1b\xea\x60\x17\xf1\x3d\x1f\xac\x35\x36\x01\x3f\xa5\x73\x35\xd7\x7d\x42\xae\xd0\x12\xb4\xcf\xb9\x6b\xca\x12\x9d\x83\x16\xe4\x97\xd4\x42\x96\x9c\x8c\x85\x38\x81\xf3\xa6\xae\xd1\x96\xdc\x21\x2b\x9a\x3a\xcf\x62\x8e\xa9\x32\x12\xa5\xde\x84\x29\x62\x17\xed\x97\x90\xc2\x7c\xe8\x2b\xa0\x27\xdd\x27\xfc\x8d\x9b\x38\x0e\xfd\xd6\x98\x9c\x83\x86\x52\x38\xd7\xd1\x49\x42\xf5\x32\x8f\xb2\x38\x11\xd2\xd9\xa6\x9a\xf3\x59\x15\x57\x6b\x1e\x43\xb5\x30\x27\xd0\x23\xd4\x14\xce\x29\x6d\x4a\xc6\x9a\x8b\x2d\x42\xfb\x9c\xd7\x56\xee\xb8\x3d\xb0\x28\xc9\x36\x5f\x27\xc9\xb8\x63\x3b\xc3\x2b\x57\x0d\x86\xc0\x2e\x91\x01\x17\x08\x39\x61\x8f\x88\x81\xd1\xf0\x7d\x95\x27\xcf\x2e\x30\x90\x67\x2d\x79\xc5\x0d\x52\xf0\x7e\x63\xec\x8e\x53\xdc\x28\x8e\xf8\xae\xee\x89\x7a\x92\x3a\xda\x2e\xcc\xe1\x95\x38\xbe\xbf\x1e\xe7\xa4\x2e\x31\x9d\xcf\x76\x4d\x85\x00\x7c\x6b\x6e\x94\x6b\xf2\x1d\xb9\x3a\xe1\xf0\xfd\x89\x9b\xd0\xf9\x11\xf1\x56\xb8\x3f\x1e\xce\x21\xf7\xa3\x0d\x32\xb5\x32\xa1\x34\x2a\xc2\x2d\xd9\x5f\x13\x75\x3f\x1b\x70\xc7\x1d\x6c\x71\x2b\x1d\xc5\x31\xfe\x0e\xfe\xa0\xde\x81\xd6\x06\xfa\x1a\x57\xba\x4e\x3f\x20\xc9\x9f\x43\xda\xfd\x0f\x6e\xb5\xd4\x5b\x56\x4c\x55\x99\x67\x42\xbe\x0f\x3f\xe6\x9d\xa9\x3f\xfe\x0e\x00\x00\xff\xff\xf7\x7e\x17\x21\x2b\x09\x00\x00") func pkgUiTemplatesStoresHtmlBytes() ([]byte, error) { return bindataRead( @@ -340,7 +340,7 @@ func pkgUiTemplatesStoresHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/stores.html", size: 2415, mode: os.FileMode(436), modTime: time.Unix(1573923761, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/stores.html", size: 2347, mode: os.FileMode(420), modTime: time.Unix(1574377450, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -360,7 +360,7 @@ func pkgUiStaticCssAlertsCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/alerts.css", size: 401, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/alerts.css", size: 401, mode: os.FileMode(420), modTime: time.Unix(1575919484, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -380,7 +380,7 @@ func pkgUiStaticCssGraphCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/graph.css", size: 3844, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/graph.css", size: 3844, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -400,7 +400,7 @@ func pkgUiStaticCssPrometheusCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/prometheus.css", size: 470, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/prometheus.css", size: 470, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -420,7 +420,7 @@ func pkgUiStaticCssRulesCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/rules.css", size: 195, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/rules.css", size: 195, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -440,7 +440,7 @@ func pkgUiStaticImgAjaxLoaderGif() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -460,12 +460,12 @@ func pkgUiStaticImgFaviconIco() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/img/favicon.ico", size: 15886, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/img/favicon.ico", size: 15886, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _pkgUiStaticJsAlertsJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x54\x4d\xca\xdb\x30\x10\xdd\xfb\x14\x53\x13\xb0\xb4\xb0\xe8\x3e\x64\x55\xba\xe8\x29\x8a\x90\x26\x96\xf8\xd4\x91\x91\x14\x27\xa5\xf8\xee\xc5\xff\x36\xb1\x43\xb2\xf1\x46\x6f\xde\xbc\x9f\xc1\xd7\x1b\xa9\x64\x3d\x81\x25\x9b\x18\x87\x7f\x19\xc0\x89\xe5\x42\x3a\x0c\xe9\xb7\x41\xa9\x31\xe4\x5c\x28\x67\xd5\x17\x9b\xc0\x23\x0e\xa0\x91\x01\xf0\x51\x4b\xd2\x18\x7e\x29\x4f\x70\x81\x13\x4b\xc6\x46\x2e\xae\x96\x34\xcb\xad\xb0\xca\x53\xa9\x0c\x36\xc1\x53\xa9\xfd\x9d\x72\x7e\xee\x67\xed\x15\xd8\x7a\x56\x38\xa4\x2a\x19\xf8\x76\xb9\xc0\xf7\x69\x01\x6c\xe8\x45\xc0\x3f\xbe\xc1\x1f\x4e\xc6\xc8\xf2\x1d\x66\x21\xb5\xde\x7b\xbd\xd5\xd3\xd6\x16\xd0\x45\x9c\xd9\x3b\x03\xca\x3b\x27\xeb\xf8\x9e\x83\x85\x09\xb6\x83\x2f\xb4\x75\x33\x47\xca\xd6\x89\xb4\xfd\x77\x5a\x4f\xf8\x48\x8c\x8b\xe4\xab\xca\x21\xeb\x21\x2d\x3f\x67\x43\x3f\xda\x36\x22\x1a\x7f\x2f\x25\x91\x4f\xb2\x2b\x25\x1e\xd7\xa4\x3c\xc5\x04\x49\x86\x0a\xd3\x4f\xd7\x5b\x2c\xf6\x18\x8a\x51\xc8\x80\xb7\x73\x1c\xe3\x20\x17\xca\x58\xa7\x03\x12\x2b\x6c\x31\x68\x19\x7a\xec\xa0\xc2\xc8\x38\x38\x2c\x2a\xf7\xb7\x36\xbd\xcd\x1b\x29\x83\xea\x0b\x75\xc1\x97\x4a\xd7\x17\xb6\xd5\xdf\x09\x62\x73\xbe\x07\xb8\xe5\x2a\x9f\xe1\x07\x52\x37\xe5\xec\xcb\x5b\x0a\x5a\xbd\xf7\xaf\xc5\x6a\xc3\xc4\xbf\x42\xdb\x58\xce\x24\x9b\x1b\x7b\x19\xcc\xc8\xfc\x4e\x28\xc6\x6a\xfc\x20\x94\x67\xf8\x87\xa1\x8c\xd2\x76\x03\x59\x05\xb6\x13\xca\x86\x70\x27\x97\xf1\x84\xdb\x2c\x3b\xb1\xee\x7f\xc3\xcf\xd9\xff\x00\x00\x00\xff\xff\x96\xfe\x01\x29\x80\x04\x00\x00") +var _pkgUiStaticJsAlertsJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x96\x51\x6b\xdb\x30\x10\xc7\xdf\xf3\x29\x54\x2d\x60\x99\x25\x66\xcf\x2b\x19\x8c\x6d\x65\x7d\x1b\x74\xef\x45\x95\x2e\xf6\x51\x55\x32\x92\x9c\xb6\x8c\x7c\xf7\x21\x3b\xb6\x25\xc7\x69\x13\xea\x97\x16\x74\xa7\xfb\xfd\xef\xfe\x3e\x67\xdb\x68\xe1\xd1\x68\x82\x1a\x3d\xcb\xc9\xbf\x05\x21\x84\x2c\x19\x2d\xb8\x02\xeb\xef\x2b\xe0\x12\x2c\xcd\x0b\xa1\x50\x3c\xb2\x3e\x7c\x88\x0c\xcf\x8e\x5b\x02\x2f\x35\xd7\x12\xec\xad\x30\x9a\x6c\xc8\x92\xf9\x0a\x5d\x5e\x6c\x51\x4b\x46\xb1\x40\x61\xf4\x5a\x54\xb0\xb3\x46\xaf\xa5\x79\xd6\x34\xbf\x1e\xf2\x71\x4b\x58\x9c\x5f\x28\xd0\xa5\xaf\xc8\xd5\x66\x43\xbe\xc4\x85\xc2\x93\x04\x5a\x78\x32\x3b\xf8\xa1\xb8\x73\x8c\xce\xd4\x28\xb8\x94\x73\xa7\x4d\x1d\xd7\xdf\x13\x50\x0e\x26\x75\x82\x28\x61\x94\xe2\xb5\x3b\x4f\x55\x7a\x67\x78\x92\xf4\x37\x58\x43\xe6\x29\xd2\x69\xaf\xf6\xc3\x7f\x3d\x8c\x86\x17\xcf\xf2\xc2\x9b\xb2\x54\xc0\x0e\xa1\xfb\xfc\x7a\xd1\x4f\x52\xe2\xae\x70\x95\x79\x5e\x73\xad\x8d\xe7\x61\x7c\xee\xed\x81\x0a\xa3\x9d\x27\x9e\xdb\x12\xfc\x2f\xd5\x0a\xcf\xe6\x6e\xc9\x22\xb0\x2e\x07\x87\x46\x1d\x92\xf3\x42\x54\xa8\xa4\x05\xcd\x32\xcc\x7a\xaa\x7e\xea\x21\xbc\xa8\xb8\xeb\x94\x67\xa5\x7a\xad\xab\x56\x7e\xa3\x45\x05\xe2\x11\x64\x96\x4f\x0d\x30\x7a\x33\xd5\x13\xe0\xd8\x64\x02\xb3\xb1\xa3\xa7\xe7\x53\x4e\xa0\x27\x03\x9c\x47\x1d\x87\x18\x9d\xb7\xa7\xd9\xa4\x4a\x5f\x23\xca\x40\xb7\x1e\x2e\x3a\xf2\xe6\x9b\xcd\x3a\x54\x38\xb7\x51\x15\x4a\xb8\xb0\x51\xf3\x29\x17\x36\xea\x80\x39\xdb\xa4\xa8\x89\x27\x1a\x95\x5c\x7a\xa2\x57\xa9\xf5\x87\xd5\x26\xd1\xd5\x8a\xbf\x7e\x0f\xfa\x1c\x6b\x65\xde\x79\xee\x61\x45\x5c\x65\x1a\x25\x7f\x76\xe7\x71\xff\x96\x8c\x7e\x6a\x03\xdd\x5f\xfe\xa0\x80\x7c\x23\xfe\xc1\xc8\xd7\xf0\xd7\x16\x94\x7c\x26\xe3\x2d\x79\x01\x5c\x54\xe3\x8b\x74\xbf\x0a\xef\x82\xe7\xa8\xc1\x1e\x8f\x64\x3c\xea\xdf\xd8\x94\x21\x52\xd3\xbf\xca\x9d\x1a\xdc\x32\x65\x04\x57\x77\xde\x58\x5e\x42\x3b\x92\x5b\xcd\x85\xc7\x1d\x74\xca\xc8\x66\xb3\x21\xd4\xdb\x06\xe8\x54\x0a\x26\x81\x34\x2f\x6a\x6e\x41\x87\xb5\x91\x6c\xa5\x2e\x28\x5e\x37\x69\xeb\x68\xab\x7a\xed\x1a\x21\xc0\x39\xba\x22\x5b\xae\x1c\x0c\x9c\xa7\x30\xff\x80\x96\xa8\xcb\x77\x29\xeb\x38\xee\xa3\x90\xcf\xdc\x6a\xd4\xe5\xd9\x90\x37\x68\xcf\x61\xdc\x46\x61\x1f\x45\x94\x5c\x97\x60\x8f\x08\x17\x89\x01\x6f\x50\x79\xb0\x8e\x7c\x45\x5d\x37\x3e\xec\xed\x2a\xa4\xbd\xbf\xb8\xa3\xef\x15\xf7\xde\x32\x8a\x32\x26\x0a\xdf\xb7\xce\x7a\xbf\x51\x42\x14\x3c\x48\x1a\x56\xcd\x8c\x9e\xb0\x8f\xfa\x32\xa1\x57\x53\x83\x4d\x7c\x9f\x74\xdb\x81\xbf\xf5\xf0\xc4\xe8\xb1\x85\xe9\x2a\x62\x9a\x6c\x82\x77\xac\x78\x35\x9b\x18\xad\xcf\x18\x77\xe2\xb4\xb3\x69\x13\x27\x5f\x0c\x3b\x5a\xf2\x22\xd8\xd4\x72\x67\xb3\xc6\x86\xbe\x18\x75\xb0\xe6\x09\xd2\x71\xd3\xee\x17\x8b\x25\x0b\x3f\x1e\xf3\xeb\xc5\xff\x00\x00\x00\xff\xff\x92\xa4\x88\x99\x4d\x0a\x00\x00") func pkgUiStaticJsAlertsJsBytes() ([]byte, error) { return bindataRead( @@ -480,7 +480,7 @@ func pkgUiStaticJsAlertsJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 1152, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 2637, mode: os.FileMode(420), modTime: time.Unix(1575919541, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -500,7 +500,7 @@ func pkgUiStaticJsBucketJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/bucket.js", size: 2834, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/bucket.js", size: 2834, mode: os.FileMode(420), modTime: time.Unix(1562774024, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -520,7 +520,7 @@ func pkgUiStaticJsGraphJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 37987, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 37987, mode: os.FileMode(420), modTime: time.Unix(1572558524, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -540,7 +540,7 @@ func pkgUiStaticJsGraph_templateHandlebar() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/graph_template.handlebar", size: 8984, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/graph_template.handlebar", size: 8984, mode: os.FileMode(420), modTime: time.Unix(1572558524, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -560,7 +560,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapGridCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.css", size: 37644, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.css", size: 37644, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -580,7 +580,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapGridMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.min.css", size: 28977, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.min.css", size: 28977, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -600,7 +600,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapRebootCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.css", size: 4896, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.css", size: 4896, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -620,7 +620,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapRebootMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.min.css", size: 4019, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.min.css", size: 4019, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -640,7 +640,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap.min.css", size: 140936, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap.min.css", size: 140936, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -660,7 +660,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapBundleJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.js", size: 212345, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.js", size: 212345, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -680,7 +680,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapBundleMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.min.js", size: 70966, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.min.js", size: 70966, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -700,7 +700,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.min.js", size: 51039, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.min.js", size: 51039, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -720,7 +720,7 @@ func pkgUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs() (*asset, err return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 11273, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 11273, mode: os.FileMode(420), modTime: time.Unix(1569428366, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -740,7 +740,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsCssBootstrapGlyphiconsCss() (*asset, e return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.css", size: 14523, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.css", size: 14523, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -760,7 +760,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsCssBootstrapGlyphiconsMinCss() (*asset return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css", size: 11830, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css", size: 11830, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -780,7 +780,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Eot() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.eot", size: 98620, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.eot", size: 98620, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -800,7 +800,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Svg() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.svg", size: 507478, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.svg", size: 507478, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -820,7 +820,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Ttf() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.ttf", size: 98384, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.ttf", size: 98384, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -840,7 +840,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Woff() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff", size: 63712, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff", size: 63712, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -860,7 +860,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Woff2() (*a return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff2", size: 54420, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff2", size: 54420, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -880,7 +880,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Eot() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.eot", size: 31156, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.eot", size: 31156, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -900,7 +900,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Svg() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.svg", size: 107199, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.svg", size: 107199, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -920,7 +920,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Ttf() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.ttf", size: 30928, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.ttf", size: 30928, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -940,7 +940,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Woff() (*a return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff", size: 14712, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff", size: 14712, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -960,7 +960,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Woff2() (* return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff2", size: 12220, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff2", size: 12220, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -980,7 +980,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Eot() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.eot", size: 102152, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.eot", size: 102152, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1000,7 +1000,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Svg() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.svg", size: 378215, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.svg", size: 378215, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1020,7 +1020,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Ttf() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.ttf", size: 101932, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.ttf", size: 101932, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1040,7 +1040,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Woff() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff", size: 48704, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff", size: 48704, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1060,7 +1060,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Woff2() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff2", size: 38784, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff2", size: 38784, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1080,7 +1080,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.eot", size: 20127, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.eot", size: 20127, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1100,7 +1100,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.svg", size: 108738, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.svg", size: 108738, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1120,7 +1120,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.ttf", size: 45404, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.ttf", size: 45404, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1140,7 +1140,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff", size: 23424, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff", size: 23424, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1160,7 +1160,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff2", size: 18028, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff2", size: 18028, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1180,7 +1180,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeCss() (*asset return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.css", size: 51062, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.css", size: 51062, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1200,7 +1200,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeLess() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.less", size: 53867, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.less", size: 53867, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1220,7 +1220,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeMinCss() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.min.css", size: 42307, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.min.css", size: 42307, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1240,7 +1240,7 @@ func pkgUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinC return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1260,7 +1260,7 @@ func pkgUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinJ return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1280,7 +1280,7 @@ func pkgUiStaticVendorFuzzyFuzzyJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/fuzzy/fuzzy.js", size: 5669, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/fuzzy/fuzzy.js", size: 5669, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1300,7 +1300,7 @@ func pkgUiStaticVendorJsJquery331MinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery-3.3.1.min.js", size: 86927, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery-3.3.1.min.js", size: 86927, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1320,7 +1320,7 @@ func pkgUiStaticVendorJsJqueryHotkeysJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.hotkeys.js", size: 4490, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.hotkeys.js", size: 4490, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1340,7 +1340,7 @@ func pkgUiStaticVendorJsJqueryMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.min.js", size: 86671, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.min.js", size: 86671, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1360,7 +1360,7 @@ func pkgUiStaticVendorJsJquerySelectionJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.selection.js", size: 12881, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.selection.js", size: 12881, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1380,7 +1380,7 @@ func pkgUiStaticVendorJsPopperMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/popper.min.js", size: 19236, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/popper.min.js", size: 19236, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1400,7 +1400,7 @@ func pkgUiStaticVendorMomentMomentTimezoneWithDataMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment-timezone-with-data.min.js", size: 184495, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment-timezone-with-data.min.js", size: 184495, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1420,7 +1420,7 @@ func pkgUiStaticVendorMomentMomentMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment.min.js", size: 51825, mode: os.FileMode(436), modTime: time.Unix(1573834182, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment.min.js", size: 51825, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1440,7 +1440,7 @@ func pkgUiStaticVendorMustacheMustacheMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1460,7 +1460,7 @@ func pkgUiStaticVendorRickshawRickshawMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1480,7 +1480,7 @@ func pkgUiStaticVendorRickshawRickshawMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1500,7 +1500,7 @@ func pkgUiStaticVendorRickshawVendorD3LayoutMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1520,7 +1520,7 @@ func pkgUiStaticVendorRickshawVendorD3V3Js() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(436), modTime: time.Unix(1567289074, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/pkg/ui/rule.go b/pkg/ui/rule.go index 12188df9d9..fa068eac9d 100644 --- a/pkg/ui/rule.go +++ b/pkg/ui/rule.go @@ -7,7 +7,6 @@ import ( "net/http" "path" "regexp" - "sort" "time" "github.com/go-kit/kit/log" @@ -114,17 +113,21 @@ func ruleTmplFuncs(queryURL string) template.FuncMap { } func (ru *Rule) alerts(w http.ResponseWriter, r *http.Request) { - alerts := ru.ruleManager.AlertingRules() - alertsSorter := byAlertStateAndNameSorter{alerts: alerts} - sort.Sort(alertsSorter) + var groups []thanosrule.Group + for _, group := range ru.ruleManager.RuleGroups() { + if group.HasAlertingRules() { + groups = append(groups, group) + } + } alertStatus := AlertStatus{ - AlertingRules: alertsSorter.alerts, + Groups: groups, AlertStateToRowClass: map[rules.AlertState]string{ rules.StateInactive: "success", rules.StatePending: "warning", rules.StateFiring: "danger", }, + Counts: alertCounts(groups), } prefix := GetWebPrefix(ru.logger, ru.flagsMap, r) @@ -161,24 +164,30 @@ func (ru *Rule) Register(r *route.Router, ins extpromhttp.InstrumentationMiddlew // AlertStatus bundles alerting rules and the mapping of alert states to row classes. type AlertStatus struct { - AlertingRules []thanosrule.AlertingRule + Groups []thanosrule.Group AlertStateToRowClass map[rules.AlertState]string + Counts AlertByStateCount } -type byAlertStateAndNameSorter struct { - alerts []thanosrule.AlertingRule -} - -func (s byAlertStateAndNameSorter) Len() int { - return len(s.alerts) -} - -func (s byAlertStateAndNameSorter) Less(i, j int) bool { - return s.alerts[i].State() > s.alerts[j].State() || - (s.alerts[i].State() == s.alerts[j].State() && - s.alerts[i].Name() < s.alerts[j].Name()) +type AlertByStateCount struct { + Inactive int32 + Pending int32 + Firing int32 } -func (s byAlertStateAndNameSorter) Swap(i, j int) { - s.alerts[i], s.alerts[j] = s.alerts[j], s.alerts[i] +func alertCounts(groups []thanosrule.Group) AlertByStateCount { + result := AlertByStateCount{} + for _, group := range groups { + for _, alert := range group.AlertingRules() { + switch alert.State() { + case rules.StateInactive: + result.Inactive++ + case rules.StatePending: + result.Pending++ + case rules.StateFiring: + result.Firing++ + } + } + } + return result } diff --git a/pkg/ui/static/js/alerts.js b/pkg/ui/static/js/alerts.js index e6a52fc8e5..9d9df42af0 100644 --- a/pkg/ui/static/js/alerts.js +++ b/pkg/ui/static/js/alerts.js @@ -1,31 +1,66 @@ function init() { - $(".alert_header").click(function() { - var expanderIcon = $(this).find("i.icon-chevron-down"); - if (expanderIcon.length !== 0) { - expanderIcon.removeClass("icon-chevron-down").addClass("icon-chevron-up"); - } else { - var collapserIcon = $(this).find("i.icon-chevron-up"); - collapserIcon.removeClass("icon-chevron-up").addClass("icon-chevron-down"); - } - $(this).next().toggle(); - }); + $(".alert_header").click(function() { + var expanderIcon = $(this).find("i.icon-chevron-down"); + if (expanderIcon.length !== 0) { + expanderIcon.removeClass("icon-chevron-down").addClass("icon-chevron-up"); + } else { + var collapserIcon = $(this).find("i.icon-chevron-up"); + collapserIcon.removeClass("icon-chevron-up").addClass("icon-chevron-down"); + } + $(this).next().toggle(); + }); + + $("div.show-annotations").click(function() { + const targetEl = $('div.show-annotations'); + const icon = $(targetEl).children('i'); - $("div.show-annotations").click(function() { - const targetEl = $('div.show-annotations'); - const icon = $(targetEl).children('i'); + if (icon.hasClass('glyphicon-unchecked')) { + $(".alert_annotations").show(); + $(".alert_annotations_header").show(); + $(targetEl).children('i').removeClass('glyphicon-unchecked').addClass('glyphicon-check'); + targetEl.addClass('is-checked'); + } else if (icon.hasClass('glyphicon-check')) { + $(".alert_annotations").hide(); + $(".alert_annotations_header").hide(); + $(targetEl).children('i').removeClass('glyphicon-check').addClass('glyphicon-unchecked'); + targetEl.removeClass('is-checked'); + } + }); + + function displayAlerts(alertState, shouldDisplay) { + $("#alertsTable > tbody > tr." + alertState).each(function(_, container) { + $(container).toggle(shouldDisplay); + }); + } - if (icon.hasClass('glyphicon-unchecked')) { - $(".alert_annotations").show(); - $(".alert_annotations_header").show(); - $(targetEl).children('i').removeClass('glyphicon-unchecked').addClass('glyphicon-check'); - targetEl.addClass('is-checked'); - } else if (icon.hasClass('glyphicon-check')) { - $(".alert_annotations").hide(); - $(".alert_annotations_header").hide(); - $(targetEl).children('i').removeClass('glyphicon-check').addClass('glyphicon-unchecked'); - targetEl.removeClass('is-checked'); + if(localStorage.hideInactiveAlerts === "true") { + $("#inactiveAlerts").parent().removeClass("active"); + displayAlerts("alert-success", false); } - }); + if(localStorage.hidePendingAlerts === "true") { + $("#pendingAlerts").parent().removeClass("active"); + displayAlerts("alert-warning", false); + } + if(localStorage.hideFiringAlerts === "true") { + $("#firingAlerts").parent().removeClass("active"); + displayAlerts("alert-danger", false); + } + + $("#alertFilters :input").change(function() { + const target = $(this).attr("id"); + var shouldHide = $(this).parent().hasClass("active"); + if (target === "inactiveAlerts") { + localStorage.setItem("hideInactiveAlerts", shouldHide); + displayAlerts("alert-success", !shouldHide); + } else if (target === "pendingAlerts") { + localStorage.setItem("hidePendingAlerts", shouldHide); + displayAlerts("alert-warning", !shouldHide); + } else if (target === "firingAlerts") { + localStorage.setItem("hideFiringAlerts", shouldHide); + displayAlerts("alert-danger", !shouldHide); + } + }); + } $(init); diff --git a/pkg/ui/templates/alerts.html b/pkg/ui/templates/alerts.html index a7cfed48c3..21c52117ba 100644 --- a/pkg/ui/templates/alerts.html +++ b/pkg/ui/templates/alerts.html @@ -1,75 +1,94 @@ {{define "head"}} - - + + {{end}} {{define "content"}} -

-

Alerts

-
- - -
- - - {{$alertStateToRowClass := .AlertStateToRowClass}} - {{range .AlertingRules}} - {{$activeAlerts := .ActiveAlerts}} - - - - - +
{{.Name}} ({{len $activeAlerts}} active)
-
-
{{.HTMLSnippet queryURL}}
-
- {{if $activeAlerts}} - - - - - - - - {{range $activeAlerts}} - - - - {{else}} - - - - {{end}} - -
LabelsStateActive SinceValue
- {{range $label, $value := .Labels.Map}} - {{$label}}="{{$value}}" +
+

Alerts

+
+ + + +
+
+
+ + +
+ + + {{$alertStateToRowClass := .AlertStateToRowClass}} + {{range .Groups}} + + + + {{range .AlertingRules}} + {{$activeAlerts := .ActiveAlerts}} + + + + + + {{end}} - - - - - - {{ if .Annotations.Map}} - - - - - - - {{end}} + {{else}} + + + {{end}} -
+ {{.OriginalFile}} > {{.Name}} +
{{.Name}} ({{len $activeAlerts}} active)
+
+
{{.HTMLSnippet pathPrefix}}
+
+ {{if $activeAlerts}} + + + + + + + + {{range $activeAlerts}} + + + + + + + {{ if .Annotations.Map}} + + + + + + + {{end}} + {{end}} +
LabelsStateActive SinceValue
+ {{range $label, $value := .Labels.Map}} + {{$label}}="{{$value}}" + {{end}} + {{.State}}{{.ActiveAt.UTC}}{{.Value}}
+ {{end}} +
{{.State}}{{.ActiveAt.UTC}}{{.Value}}
+ No alerting rules defined +
- {{end}} -
- No alerting rules defined -
- +
+
{{end}} From fd9e2212ba1530774ac766eab7139b901f227e1a Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 10 Dec 2019 09:30:43 -0500 Subject: [PATCH 097/257] use thanos website as help link (#1863) Signed-off-by: yeya24 --- pkg/ui/bindata.go | 16 ++++++++-------- pkg/ui/templates/bucket_menu.html | 2 +- pkg/ui/templates/query_menu.html | 2 +- pkg/ui/templates/rule_menu.html | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/ui/bindata.go b/pkg/ui/bindata.go index 27db965494..d9cd32f615 100644 --- a/pkg/ui/bindata.go +++ b/pkg/ui/bindata.go @@ -180,7 +180,7 @@ func pkgUiTemplatesAlertsHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 5108, mode: os.FileMode(420), modTime: time.Unix(1575922733, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 5108, mode: os.FileMode(420), modTime: time.Unix(1575983680, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -205,7 +205,7 @@ func pkgUiTemplatesBucketHtml() (*asset, error) { return a, nil } -var _pkgUiTemplatesBucket_menuHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x6c\x92\x4f\x8f\x9c\x30\x0c\xc5\xef\xf3\x29\xac\xf4\x9c\xe6\x5e\x0d\x73\xe8\xa9\xc7\x1e\x56\xbd\x56\x86\x18\xb0\x26\xe3\xa0\x60\xe8\x54\x88\xef\x5e\x85\x3f\x5b\x86\xdd\x13\xe4\xe9\xd9\xcf\x3f\x27\xd3\xe4\xa9\x66\x21\x30\x82\xa3\x99\xe7\xcb\x55\x70\x84\x2a\x60\xdf\x17\x59\x2a\x31\x41\xcd\x4f\xf2\x56\x63\x07\xab\x60\xe9\xd9\xa1\x78\xdb\x3f\x76\xc1\x63\xba\x43\xd9\x2c\x5f\x73\xbb\x00\x00\x5c\x3d\xbf\xf7\xa9\xa2\x28\xb2\x50\xb2\x75\x18\xd8\x6f\x8e\xc5\x55\x0e\xaa\x51\x40\xff\x76\x54\x98\xf5\x60\x5e\xe3\xad\xc6\xa6\x09\x94\x0c\x78\x54\xdc\x4e\xb9\x67\x08\xd8\xf5\xb4\xcb\x98\x1a\xd2\xc2\x7c\x11\x1c\x6d\xce\x23\x51\x03\x98\x18\xb7\x69\xc9\x17\xa6\xc6\x90\x0b\x16\x35\x7b\x52\x0c\x6b\xcc\xa9\x22\x60\x49\xa1\x30\x6f\x4b\x54\x66\xe4\x06\x95\xa3\x1c\x06\x5f\x86\xef\x3b\x94\xcf\x87\xb5\x5c\x65\xfb\xd5\x65\xcb\x01\xd7\xad\x88\x07\x05\x4f\x0d\xca\x84\xe2\x0d\xb4\x89\xea\xc2\x4c\x13\x74\xa8\xed\xcf\x44\x35\x3f\x61\x9e\x9d\xb9\xbd\xb5\x28\xb1\x87\xef\x43\x75\x27\x85\x5f\x4c\x7f\x28\x5d\x1d\x1e\x3a\xe6\xc5\xb3\x3f\x71\xbd\x86\xec\xcb\x83\xf7\x2d\x9e\xc8\x86\x70\xaa\xc8\xaf\xe3\xd5\xb3\xf8\x02\x1f\x7c\x96\x95\x1e\x9f\xb8\x3e\x70\xda\xc0\x72\xdf\x19\x5b\xd5\xae\xff\xe6\x5c\xc3\xda\x0e\xe5\xd7\x2a\x3e\x9c\x2e\x8c\x96\xe3\xf6\x67\x60\xbf\xde\xdf\x65\x40\xb9\x9b\xdb\x0f\x0a\xdd\x0b\xf5\xff\x0d\x07\x3e\xb1\xb8\x21\x1c\x6f\xc0\xf3\xb8\xbd\xd0\xf5\xf7\xea\x04\xc7\xdb\x65\x9a\x48\xfc\x3c\x5f\xfe\x05\x00\x00\xff\xff\xdb\x9d\xc3\xee\x11\x03\x00\x00") +var _pkgUiTemplatesBucket_menuHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x6c\x92\x31\x8f\x9c\x40\x0c\x85\xfb\xfd\x15\xd6\xa4\x9e\x9b\x3e\x82\x2d\x52\xa5\x4c\x71\x4a\x1b\x99\x1d\xc3\x5a\xeb\x35\x68\x30\x64\x23\xc4\x7f\x8f\x06\x96\x0b\x4b\xae\x82\x79\x7a\xf6\xf3\xe7\x99\x69\x8a\x54\xb3\x12\x38\xc5\xd1\xcd\xf3\xa9\x50\x1c\xe1\x22\xd8\xf7\x65\x96\x2a\x4c\x50\xf3\x83\xa2\xb7\xb6\x83\x55\xf0\xf4\xe8\x50\xa3\xef\xef\x9b\x10\x31\xdd\xa0\x6a\x96\xaf\x3b\x9f\x00\x00\x8a\xc8\x1f\x7d\x2e\xad\x1a\xb2\x52\xf2\xb5\x0c\x1c\x9f\x8e\xc5\x55\x0d\x66\xad\x82\xfd\xe9\xa8\x74\xeb\xc1\xbd\xc6\x7b\x6b\x9b\x46\x28\x39\x88\x68\xf8\x3c\xe5\x9e\x22\xd8\xf5\xb4\xc9\x98\x1a\xb2\xd2\x7d\x51\x1c\x7d\xce\x23\x35\x07\x98\x18\x9f\xd3\x52\x2c\x5d\x8d\x92\x0b\x16\x35\x7b\x52\x2b\x6b\xcc\xa1\x42\xb0\x22\x29\xdd\xfb\x12\x95\x19\xb9\x41\xe3\x56\x77\x83\x2f\xc3\xf7\x1d\xea\xe7\xc3\x7a\xbe\x64\x7b\x11\xb2\x65\x87\x1b\x56\xc4\x9d\x82\x87\x06\x55\x42\x8d\x0e\xae\x89\xea\xd2\x4d\x13\x74\x68\xd7\x1f\x89\x6a\x7e\xc0\x3c\x07\x77\x7e\xbf\xa2\xb6\x3d\x7c\x1b\x2e\x37\x32\xf8\xc9\xf4\x9b\x52\x11\x70\xd7\x31\x2f\x9e\xe3\x81\xeb\x35\x64\x5b\x1e\x7c\x6c\xf1\x40\x36\xc8\xa1\x22\xbf\x8e\x57\xcf\xe2\x13\xde\xf9\x3c\x1b\xdd\x3f\x71\xfd\xc7\xe9\x85\xf5\xb6\x31\x5e\xcd\xba\xfe\x6b\x08\xb6\x80\xbd\x71\x1b\x1a\x32\x63\x6d\x7c\x6f\x98\x8c\xe2\xdb\x3d\x06\x07\xdb\x05\xff\xaa\x04\xf5\xe6\xce\xdf\x49\xba\x17\xee\x7f\x3b\x16\x3e\xd0\x84\x41\xf6\x77\x10\x79\x7c\xbe\xd1\xf5\xb7\x08\x8a\xe3\xf9\x34\x4d\xa4\x71\x9e\x4f\x7f\x03\x00\x00\xff\xff\x74\xc4\xee\x39\x13\x03\x00\x00") func pkgUiTemplatesBucket_menuHtmlBytes() ([]byte, error) { return bindataRead( @@ -220,7 +220,7 @@ func pkgUiTemplatesBucket_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 785, mode: os.FileMode(420), modTime: time.Unix(1564066864, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 787, mode: os.FileMode(420), modTime: time.Unix(1575985294, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -245,7 +245,7 @@ func pkgUiTemplatesGraphHtml() (*asset, error) { return a, nil } -var _pkgUiTemplatesQuery_menuHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x54\xc1\x8e\xd4\x30\x0c\x3d\xb3\x5f\x61\x65\x25\x6e\xa5\x77\x68\x7b\xe0\x02\xdc\x10\xbb\x77\xe4\x4e\xdc\x36\x9a\x34\x89\x12\x67\x18\x54\xf5\xdf\x51\x3a\x33\x25\xed\xec\x48\x1c\x38\xa5\xb1\x9c\xe7\xf7\xec\xe7\x4e\x93\xa4\x4e\x19\x02\x61\xf0\x24\xe6\xf9\x09\x00\xa0\x32\x78\x82\x83\xc6\x10\xea\x14\x6e\xd1\x43\xa7\xce\x24\x0b\xb6\x0e\x2e\x81\x82\xce\x0e\x8d\x2c\xc2\x78\x0b\x48\xf4\x47\x68\xfb\xe5\x14\xcd\x82\x03\x50\x49\xb5\x22\x1d\xac\x61\x54\x86\x7c\xd1\xe9\xa8\xe4\x9a\x03\x50\xb5\x91\xd9\x1a\xe0\xdf\x8e\x6a\x71\xb9\x88\x2d\x81\x82\x6d\xdf\x6b\xf2\x02\x24\x32\x5e\x6f\x09\x53\x6b\x74\x81\x6e\x61\xf4\x3d\x71\x2d\x9e\x0d\x9e\x8a\x54\x8f\x0c\x0b\x40\xaf\xf0\xca\x97\x64\x2d\x3a\xd4\xe9\xc1\x12\x4d\x39\xde\xea\x4b\x99\xdd\x0b\x8d\x2d\xe9\x5a\xbc\x2e\xa5\x92\x4a\xd5\x23\x2b\x6b\x32\xe2\x00\x55\x70\x68\xde\xa6\x5a\xa8\x43\x4a\xae\xca\x94\x92\x89\x2d\x2f\x02\xb3\x08\xee\x00\x5a\x8f\x46\x0a\x18\x3c\x75\xb5\x98\x26\x70\xc8\xc3\x77\x4f\x9d\x3a\xc3\x3c\x97\xa2\x79\x1d\xd0\xd8\x50\x95\x98\x61\xa4\x46\x2b\xb9\xd3\xb1\x85\xbd\x35\x0b\xd6\xae\x6d\x94\x44\xbd\xcb\x4f\x8e\xc8\x33\x00\x2a\xad\xb2\x9c\x42\x31\x8d\xa2\xd9\xd0\x2f\xb4\x32\xc7\x87\xd4\x7b\x8f\x6e\x10\xcd\x97\x74\x24\xfa\x55\xa9\xd5\xff\xad\x10\xd8\x7a\x0a\xa2\x79\x59\xce\xbf\x35\xde\x3d\x00\x07\xe9\xad\x93\xf6\x97\xd9\x29\x5d\xa6\x72\xa9\xf1\x2c\xf6\xd5\xd7\x47\xd7\x51\xef\x4c\xb9\x42\x82\xb7\x3a\x33\xf4\xe2\xaa\x01\x83\xb3\x2e\xba\x5a\xb0\x8f\xf4\xc0\x9c\xcd\x0b\x23\xc7\xb0\x75\xd7\x01\x3d\xf1\xea\xa7\xcd\xf4\xef\x97\x6d\x65\x38\x92\x89\x77\xda\x72\xcf\xad\x99\x4b\xaf\x1f\xf7\x35\x11\x12\xcd\x8f\x68\x58\x8d\x04\xef\x71\x74\x9f\xe0\x73\x54\x5a\xc2\x37\xd3\x59\x3f\x2e\xbb\xf1\x16\xab\x52\xaa\xd3\x6e\xc8\xff\x36\xf6\xfb\x81\x3c\x70\xc1\xc0\xec\xc2\xc7\xb2\xec\x15\x0f\xb1\xfd\x70\xb0\x63\xc9\xcb\x8a\x14\xca\x5e\xbf\x04\xdc\xfe\x0d\x3f\x5b\x8d\xe6\x28\x9a\xaf\xa4\xdd\x1d\xdd\x3d\xb3\xaa\x8c\x3a\x5f\xdd\x4c\x4a\x76\xa9\x4a\x83\xa7\xe6\x69\x9a\xc8\xc8\x79\x7e\xfa\x13\x00\x00\xff\xff\x66\x4c\xe0\x1e\x54\x05\x00\x00") +var _pkgUiTemplatesQuery_menuHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x54\x31\x8f\x9c\x3c\x10\xad\xbf\xfb\x15\x23\x9f\xf4\x75\x9c\xfb\x04\x28\xd2\x24\xe9\xa2\xdc\xf5\xd1\xb0\x1e\xc0\x5a\x63\x5b\xf6\xb0\xd9\x08\xf1\xdf\x23\xb3\xbb\xc4\xb0\xb7\x52\x8a\x54\xc8\xa3\xe7\x37\xef\xcd\x3c\x33\x4d\x8a\x5a\x6d\x09\x84\xc5\x93\x98\xe7\x27\x00\x80\xd2\xe2\x09\x0e\x06\x63\xac\x52\xb9\xc1\x00\xad\x3e\x93\x2a\xd8\x79\xb8\x14\x0a\x3a\x7b\xb4\xaa\x88\xc3\xad\xa0\x30\x1c\xa1\xe9\x96\xaf\xa8\x17\x1e\x80\x52\xe9\x95\xe9\xe0\x2c\xa3\xb6\x14\x8a\xd6\x8c\x5a\xad\x18\x80\xb2\x19\x99\x9d\x05\xfe\xe5\xa9\x12\x97\x83\xd8\x0a\x28\xd8\x75\x9d\xa1\x20\x40\x21\xe3\xf5\x94\x38\x8d\x41\x1f\xe9\x56\xc6\xd0\x11\x57\xe2\xd9\xe2\xa9\x48\xfd\xc8\xb2\x00\x0c\x1a\xaf\x7a\x49\x55\xa2\x45\x93\x2e\x2c\xd5\x84\x09\xce\x5c\xda\xec\x6e\x18\x6c\xc8\x54\xe2\x6d\x69\x95\x5c\xea\x0e\x59\x3b\x9b\x09\x07\x28\xa3\x47\xfb\xbe\xd4\x42\x1f\x12\xb8\x94\x09\x92\x99\x95\x17\x83\x59\x05\x77\x04\x4d\x40\xab\x04\xf4\x81\xda\x4a\x4c\x13\x78\xe4\xfe\x5b\xa0\x56\x9f\x61\x9e\xa5\xa8\xdf\x7a\xb4\x2e\x96\x12\x33\x8e\x34\x68\xad\x76\x3e\xb6\xb4\xb7\x61\xc1\x3a\xb5\x8d\x93\xd1\xec\xf0\x29\x11\x39\x02\xa0\x34\x3a\xc3\x14\x9a\x69\x10\xf5\x46\x7e\x61\xb4\x3d\x3e\x94\xde\x05\xf4\xbd\xa8\x3f\xa7\x4f\x92\x5f\x4a\xa3\xff\x6d\x87\xc8\x2e\x50\x14\xf5\xeb\xf2\xfd\xd3\xe3\xbf\x07\xe4\xa0\x82\xf3\xca\xfd\xb4\x3b\xa7\xcb\x56\x2e\x3d\x9e\xc5\xbe\xfb\x7a\xe9\xba\xea\x5d\x28\x57\x4a\x08\xce\x64\x81\x5e\x52\xd5\x63\xf4\xce\x8f\xbe\x12\x1c\x46\x7a\x10\xce\xfa\x95\x91\xc7\xb8\x4d\xd7\x01\x03\xf1\x9a\xa7\xcd\xf6\xef\x1f\xdb\xaa\x70\x20\x3b\xde\x79\xcb\x33\xb7\x22\x97\x59\x3f\x9e\x6b\x12\x24\xea\xef\xa3\x65\x3d\x10\xfc\x8f\x83\xff\x08\x9f\x46\x6d\x14\x7c\xb5\xad\x0b\xc3\xf2\x36\xde\x53\x25\x95\x3e\xed\x96\xfc\x77\x6b\xbf\x5f\xc8\x83\x14\xf4\xcc\x3e\x7e\x90\x92\x97\x77\xf1\xa2\x9d\xec\x88\x59\xdb\xae\x88\x8c\x81\x49\xbd\x0c\x4a\x0a\xb8\xfd\x1d\x7e\x34\x06\xed\x51\xd4\x5f\xc8\xf8\x3b\xc1\x7b\x6d\xa5\x1c\x4d\xfe\x78\x33\x33\xd9\xa1\x94\x16\x4f\xf5\xd3\x34\x91\x55\xf3\xfc\xf4\x3b\x00\x00\xff\xff\xe6\xe9\x64\x17\x56\x05\x00\x00") func pkgUiTemplatesQuery_menuHtmlBytes() ([]byte, error) { return bindataRead( @@ -260,12 +260,12 @@ func pkgUiTemplatesQuery_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1364, mode: os.FileMode(420), modTime: time.Unix(1564066864, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1366, mode: os.FileMode(420), modTime: time.Unix(1575985293, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _pkgUiTemplatesRule_menuHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x53\x4f\xeb\x9c\x3c\x10\xbe\xef\xa7\x18\xe6\x3d\xe7\xcd\xbd\x44\xa1\xb7\x1e\x4b\xf9\xdd\xcb\xb8\x19\x75\xd8\xd9\x28\x31\xca\x16\xf1\xbb\x97\xe8\xee\x56\xa5\x3d\x14\x7a\x8a\x79\x78\xf2\xfc\x49\xc6\x79\xf6\x5c\x4b\x60\xc0\x40\x13\x2e\xcb\x05\x00\xc0\x05\x9a\xe0\xaa\x34\x0c\x45\x86\x2b\x8a\x50\xcb\x83\xbd\x49\x5d\x0f\x1b\x60\xf8\xd1\x53\xf0\x66\xb8\xbf\x00\x4f\xf1\x06\x55\xb3\xae\x58\xae\x3a\x00\xce\xcb\x5b\xe9\xda\x85\x44\x12\x38\x9a\x5a\x47\xf1\x6f\x0e\x80\xab\xc6\x94\xba\x00\xe9\x47\xcf\x05\x6e\x1b\x3c\x06\x30\xa9\x6b\x1a\xe5\x88\xe0\x29\xd1\x73\x97\x35\x55\xa9\x1f\xf8\x05\x53\x6c\x38\x15\xf8\x5f\xa0\xc9\x64\x3f\x0e\x09\x81\xa2\xd0\x33\x2f\xfb\x02\x6b\xd2\x7c\x60\x45\x33\x27\x76\xba\xd9\x9c\x4e\x28\x55\xac\x05\x7e\xac\x56\xb9\xa5\x34\x94\xa4\x0b\xbb\xe0\x00\x6e\xe8\x29\xfc\x3e\xaa\x91\x6b\x26\x3b\x9b\x29\xbb\xb2\x76\x2b\xb8\x43\xe8\x24\x50\x45\x0a\x1e\xa1\x8d\x5c\x17\x38\xcf\xd0\x53\x6a\xbf\x46\xae\xe5\x01\xcb\x62\xb1\xfc\x68\x29\x74\x83\xb3\xb4\xd3\xc8\x17\x2d\xfe\xd4\xe3\x28\xfb\xba\x2c\x78\xdf\xda\xa1\xc9\xa8\x27\x7e\x9e\x88\x3d\x03\xc0\xa9\xec\x38\x46\x12\xdf\xb1\x3c\xc4\x37\x2a\xe1\xf6\xc7\xe8\xa4\x1c\xd3\x80\xe5\xe7\x75\xcd\x05\x9c\x55\xf9\xb7\x1e\x71\x54\x1e\xb0\xfc\x96\x97\xbf\x70\x38\x30\x4e\x6f\x72\x30\x6c\x53\xea\x87\x4f\xd6\x36\x92\xda\xb1\xfa\xff\xda\xdd\x6d\x5a\xdf\xc3\x48\xf7\xfc\x42\x78\x0d\xe2\xf7\x4a\x29\xdc\xb0\xfc\xc2\xda\x1f\xde\x6b\x9b\x84\x63\x32\x67\x47\xdd\xcf\x89\x97\xe9\xfd\x1f\xfd\xda\x38\x1b\x68\x2a\x2f\xf3\xcc\xc1\x2f\xcb\xe5\x67\x00\x00\x00\xff\xff\xb2\x13\x33\x60\xc1\x03\x00\x00") +var _pkgUiTemplatesRule_menuHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x53\x4d\x8b\xdb\x30\x10\xbd\xe7\x57\x0c\xea\x59\xab\x7b\x91\x0d\xbd\xf5\x58\xca\xde\xcb\x38\x1a\x3b\x22\x93\xb1\x91\x26\x26\xc5\xf8\xbf\x17\xd9\x9b\xd4\x36\xed\xa1\xd0\x93\xd0\xe3\xf9\x7d\x68\xc6\xd3\x14\xa8\x8d\x42\x60\x04\x47\x33\xcf\x27\x00\x00\x2f\x38\xc2\x99\x31\xe7\xaa\xc0\x0d\x26\x68\xe3\x83\x82\xd5\x7e\x80\x15\xb0\xf4\x18\x50\x82\xcd\xb7\x27\x10\x30\x5d\xa1\xe9\x96\xd3\xd4\x8b\x0e\x80\x0f\xf1\xa5\x74\xee\x45\x31\x0a\x25\xdb\xf2\x3d\x86\x17\x07\xc0\x37\x77\xd5\x5e\x40\x7f\x0e\x54\x99\xf5\x62\xf6\x01\xac\xf6\x5d\xc7\x94\x0c\x04\x54\xfc\xb8\x15\x4d\x66\x1c\x32\x3d\x61\x4c\x1d\x69\x65\x3e\x09\x8e\xb6\xf8\x91\xa8\x01\x4c\x11\x3f\xf2\x52\xa8\x4c\x8b\x5c\x3e\x58\xd0\xc2\x49\x3d\xaf\x36\x87\x2f\x18\x1b\xe2\xca\xbc\x2f\x56\xa5\x65\xec\x50\x63\x2f\x9b\xe0\x00\x3e\x0f\x28\x7f\x8e\x6a\xe3\xb9\x90\xbd\x2b\x94\x4d\x59\xb7\x16\xdc\x20\x78\x10\x68\x12\x4a\x30\x70\x49\xd4\x56\x66\x9a\x60\x40\xbd\x7c\x4b\xd4\xc6\x07\xcc\xb3\x33\xf5\xfb\x05\xa5\xcf\xde\xe1\x46\xa3\x3c\x74\x0c\x87\x1e\x7b\xd9\xe7\x63\xc1\xeb\xd5\x76\x4d\xee\x7c\xe0\x97\x8d\xd8\x32\x00\x3c\xc7\x0d\xc7\x46\xa5\x9b\xa9\x77\xf1\x2d\x47\xb9\xfe\x35\x3a\x32\x25\xcd\xa6\xfe\xb2\x9c\xa5\x80\x77\x1c\xff\xaf\x47\xba\x33\x65\x53\x7f\x2f\xc7\x3f\x38\xec\x18\x87\x99\xec\x0c\x2f\xaa\x43\xfe\xec\x9c\x2e\x43\x78\x8b\xbd\xeb\x48\x35\x4a\x67\xb3\x62\x52\x0a\x6f\xb7\xe0\x0c\x3c\x57\xf1\x47\xc3\x28\x57\x53\x7f\x25\x1e\x76\x13\x5b\x77\x61\x9f\xcd\xbb\x3b\x6f\x37\x25\xc4\xf1\xf5\x27\xfd\xbe\x78\x27\x38\xd6\xa7\x69\x22\x09\xf3\x7c\xfa\x15\x00\x00\xff\xff\x4b\xdb\xe4\x09\xc3\x03\x00\x00") func pkgUiTemplatesRule_menuHtmlBytes() ([]byte, error) { return bindataRead( @@ -280,7 +280,7 @@ func pkgUiTemplatesRule_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 961, mode: os.FileMode(420), modTime: time.Unix(1564066864, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 963, mode: os.FileMode(420), modTime: time.Unix(1575985293, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -480,7 +480,7 @@ func pkgUiStaticJsAlertsJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 2637, mode: os.FileMode(420), modTime: time.Unix(1575919541, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 2637, mode: os.FileMode(420), modTime: time.Unix(1575983680, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/pkg/ui/templates/bucket_menu.html b/pkg/ui/templates/bucket_menu.html index 76060b97fe..92a8a4c4a0 100644 --- a/pkg/ui/templates/bucket_menu.html +++ b/pkg/ui/templates/bucket_menu.html @@ -8,7 +8,7 @@
diff --git a/pkg/ui/templates/query_menu.html b/pkg/ui/templates/query_menu.html index 533f9dc1ae..f65676fc98 100644 --- a/pkg/ui/templates/query_menu.html +++ b/pkg/ui/templates/query_menu.html @@ -16,7 +16,7 @@
diff --git a/pkg/ui/templates/rule_menu.html b/pkg/ui/templates/rule_menu.html index 77db75b303..98e73de5e2 100644 --- a/pkg/ui/templates/rule_menu.html +++ b/pkg/ui/templates/rule_menu.html @@ -10,7 +10,7 @@ From 51d584ed73332ceae3ced56175bd439c9927083a Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 10 Dec 2019 18:10:45 +0000 Subject: [PATCH 098/257] Be clear about downsampling process. (#1864) Previous doc was not fully correct: https://github.com/thanos-io/thanos/blob/825f119982dd2db4b9a3b4fe0a4b652331670ff2/cmd/thanos/downsample.go#L225 Signed-off-by: Bartek Plotka --- docs/components/compact.md | 5 ++++- pkg/compact/downsample/downsample.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/components/compact.md b/docs/components/compact.md index a6472c7c99..30e536ed37 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -9,7 +9,10 @@ menu: components The compactor component of Thanos applies the compaction procedure of the Prometheus 2.0 storage engine to block data stored in object storage. It is generally not semantically concurrency safe and must be deployed as a singleton against a bucket. -It is also responsible for downsampling of data - performing 5m downsampling after **40 hours** and 1h downsampling after **10 days**. +It is also responsible for downsampling of data: + +* creating 5m downsampling for blocks larger than **40 hours** (2d, 2w) +* creating 1h downsampling for blocks larger than **10 days** (2w). Example: diff --git a/pkg/compact/downsample/downsample.go b/pkg/compact/downsample/downsample.go index e64e3ceb02..5bcfccc81b 100644 --- a/pkg/compact/downsample/downsample.go +++ b/pkg/compact/downsample/downsample.go @@ -28,7 +28,7 @@ const ( ResLevel2 = int64(60 * 60 * 1000) // 1 hour in milliseconds. ) -// Downsampling ranges i.e. after what time we start to downsample blocks (in seconds). +// Downsampling ranges i.e. minimum block size after which we start to downsample blocks (in seconds). const ( DownsampleRange0 = 40 * 60 * 60 * 1000 // 40 hours. DownsampleRange1 = 10 * 24 * 60 * 60 * 1000 // 10 days. From 4506fb1201704ea59f96aca68f8d98ac9cf891b9 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Wed, 11 Dec 2019 14:07:18 -0500 Subject: [PATCH 099/257] fix /api/v1/rules value (#1872) Signed-off-by: yeya24 --- pkg/rule/api/v1.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/rule/api/v1.go b/pkg/rule/api/v1.go index bacdb5570a..7927a97d67 100644 --- a/pkg/rule/api/v1.go +++ b/pkg/rule/api/v1.go @@ -3,6 +3,7 @@ package v1 import ( "fmt" "net/http" + "strconv" "time" "github.com/NYTimes/gziphandler" @@ -141,7 +142,7 @@ type Alert struct { Annotations labels.Labels `json:"annotations"` State string `json:"state"` ActiveAt *time.Time `json:"activeAt,omitempty"` - Value float64 `json:"value"` + Value string `json:"value"` PartialResponseStrategy string `json:"partial_response_strategy"` } @@ -154,7 +155,7 @@ func rulesAlertsToAPIAlerts(s storepb.PartialResponseStrategy, rulesAlerts []*ru Annotations: ruleAlert.Annotations, State: ruleAlert.State.String(), ActiveAt: &ruleAlert.ActiveAt, - Value: ruleAlert.Value, + Value: strconv.FormatFloat(ruleAlert.Value, 'e', -1, 64), } } From 29813bae5720cddcf2698d266f1dc91b09e9b15c Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 12 Dec 2019 07:38:06 +0800 Subject: [PATCH 100/257] clean up all white noise (#1801) * clean up all white noise Signed-off-by: Xiang Dai <764524258@qq.com> * check white noise in check-docs Signed-off-by: Xiang Dai <764524258@qq.com> * add cleanup-white-noise script Signed-off-by: Xiang Dai <764524258@qq.com> * update how-to-contribute-to-docs.md Signed-off-by: Xiang Dai <764524258@qq.com> * Add format Signed-off-by: Xiang Dai <764524258@qq.com> * add check for format tag Signed-off-by: Xiang Dai <764524258@qq.com> * cleanup white noise when merge master Signed-off-by: Xiang Dai <764524258@qq.com> * feedback Signed-off-by: Xiang Dai <764524258@qq.com> --- .github/PULL_REQUEST_TEMPLATE.md | 8 ++-- MAINTAINERS.md | 10 ++--- Makefile | 22 ++++++++++- docs/components/bucket.md | 2 +- docs/components/rule.md | 22 +++++------ docs/contributing/how-to-change-go-version.md | 5 +-- .../contributing/how-to-contribute-to-docs.md | 7 +++- docs/getting-started.md | 10 ++--- docs/governance.md | 12 +++--- docs/integrations.md | 14 +++---- docs/proposals/201909_thanos_sharding.md | 38 +++++++++---------- docs/quick-tutorial.md | 16 ++++---- docs/release-process.md | 14 +++---- docs/storage.md | 10 ++--- docs/tracing.md | 4 +- scripts/cleanup-white-noise.sh | 2 + .../katacoda/thanos/1-globalview/finish.md | 6 +-- .../katacoda/thanos/1-globalview/intro.md | 14 +++---- .../thanos/1-globalview/step1-answer.md | 4 +- .../katacoda/thanos/1-globalview/step1.md | 16 ++++---- .../katacoda/thanos/1-globalview/step2.md | 18 ++++----- .../katacoda/thanos/1-globalview/step3.md | 18 ++++----- tutorials/katacoda/thanos/2-lts/intro.md | 2 +- .../thanos/3-meta-monitoring/intro.md | 2 +- .../thanos/4-cross-cluster-comm/intro.md | 2 +- .../thanos/5-remote-receiver/intro.md | 2 +- tutorials/katacoda/thanos/6-caching/intro.md | 2 +- 27 files changed, 153 insertions(+), 129 deletions(-) create mode 100755 scripts/cleanup-white-noise.sh diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1c34c5d369..c8e50ac217 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,12 +3,12 @@ about what components it touches e.g "query:" or ".*:" --> - +```yaml - alert: ThanosRuleIsDown - expr: up{app="thanos-rule"} == 0 or absent(up{app="thanos-rule"}) + expr: up{app="thanos-ruler"} == 0 or absent(up{app="thanos-ruler"}) for: 5m labels: team: TEAM @@ -81,7 +78,7 @@ For Thanos ruler we run some alerts in local Prometheus, to make sure that Thano action: 'check {{ $labels.kubernetes_pod_name }} pod in {{ $labels.kubernetes_namespace}} namespace' dashboard: RULE_DASHBOARD - alert: ThanosRuleIsDroppingAlerts - expr: rate(thanos_alert_queue_alerts_dropped_total{app="thanos-rule"}[5m]) > 0 + expr: rate(thanos_alert_queue_alerts_dropped_total{app="thanos-ruler"}[5m]) > 0 for: 5m labels: team: TEAM @@ -91,7 +88,7 @@ For Thanos ruler we run some alerts in local Prometheus, to make sure that Thano action: 'check {{ $labels.kubernetes_pod_name }} pod logs in {{ $labels.kubernetes_namespace}} namespace' dashboard: RULE_DASHBOARD - alert: ThanosRuleGrpcErrorRate - expr: rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable",app="thanos-rule"}[5m]) > 0 + expr: rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable",app="thanos-ruler"}[5m]) > 0 for: 5m labels: team: TEAM @@ -104,32 +101,71 @@ For Thanos ruler we run some alerts in local Prometheus, to make sure that Thano ## Store Gateway -``` +[embedmd]:# (../tmp/thanos-store.rules.yaml yaml) +```yaml +name: thanos-store.rules +rules: - alert: ThanosStoreGrpcErrorRate - expr: rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable",app="thanos-store"}[5m]) > 0 + annotations: + message: Thanos Store {{$labels.job}} is failing to handle {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-store.*"}[5m])) + / + sum by (job) (rate(grpc_server_started_total{job=~"thanos-store.*"}[5m])) + * 100 > 5 + ) for: 5m labels: - team: TEAM + severity: warning +- alert: ThanosStoreSeriesGateLatencyHigh annotations: - summary: Thanos Store is returning Internal/Unavailable errors - impact: Long Term Storage Prometheus queries are failing - action: Check {{ $labels.kubernetes_pod_name }} pod logs in {{ $labels.kubernetes_namespace}} namespace - dashboard: GATEWAY_URL -- alert: ThanosStoreBucketOperationsFailed - expr: rate(thanos_objstore_bucket_operation_failures_total{app="thanos-store"}[5m]) > 0 - for: 5m + message: Thanos Store {{$labels.job}} has a 99th percentile latency of {{ $value + }} seconds for store series gate requests. + expr: | + ( + histogram_quantile(0.9, sum by (job, le) (thanos_bucket_store_series_gate_duration_seconds_bucket{job=~"thanos-store.*"})) > 2 + and + sum by (job) (rate(thanos_bucket_store_series_gate_duration_seconds_count{job=~"thanos-store.*"}[5m])) > 0 + ) + for: 10m labels: - team: TEAM + severity: warning +- alert: ThanosStoreBucketHighOperationFailures annotations: - summary: Thanos Store is failing to do bucket operations - impact: Long Term Storage Prometheus queries are failing - action: Check {{ $labels.kubernetes_pod_name }} pod logs in {{ $labels.kubernetes_namespace}} namespace - dashboard: GATEWAY_URL + message: Thanos Store {{$labels.job}} Bucket is failing to execute {{ $value | + humanize }}% of operations. + expr: | + ( + sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-store.*"}[5m])) + / + sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~"thanos-store.*"}[5m])) + * 100 > 5 + ) + for: 15m + labels: + severity: warning +- alert: ThanosStoreObjstoreOperationLatencyHigh + annotations: + message: Thanos Store {{$labels.job}} Bucket has a 99th percentile latency of + {{ $value }} seconds for the bucket operations. + expr: | + ( + histogram_quantile(0.9, sum by (job, le) (thanos_objstore_bucket_operation_duration_seconds_bucket{job=~"thanos-store.*"})) > 15 + and + sum by (job) (rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~"thanos-store.*"}[5m])) > 0 + ) + for: 10m + labels: + severity: warning ``` ## Sidecar -``` +[//]: # "TODO(kakkoyun): Generate sidecar rules using thanos-mixin." + +```yaml - alert: ThanosSidecarPrometheusDown expr: thanos_sidecar_prometheus_up{name="prometheus"} == 0 for: 5m @@ -164,15 +200,229 @@ For Thanos ruler we run some alerts in local Prometheus, to make sure that Thano ## Query +[embedmd]:# (../tmp/thanos-querier.rules.yaml yaml) +```yaml +name: thanos-querier.rules +rules: +- alert: ThanosQuerierHttpRequestQueryErrorRateHigh + annotations: + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + }}% of "query" requests. + expr: | + ( + sum(rate(http_requests_total{code=~"5..", job=~"thanos-querier.*", handler="query"}[5m])) + / + sum(rate(http_requests_total{job=~"thanos-querier.*", handler="query"}[5m])) + ) * 100 > 5 + for: 5m + labels: + severity: critical +- alert: ThanosQuerierHttpRequestQueryRangeErrorRateHigh + annotations: + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + }}% of "query_range" requests. + expr: | + ( + sum(rate(http_requests_total{code=~"5..", job=~"thanos-querier.*", handler="query_range"}[5m])) + / + sum(rate(http_requests_total{job=~"thanos-querier.*", handler="query_range"}[5m])) + ) * 100 > 5 + for: 5m + labels: + severity: critical +- alert: ThanosQuerierGrpcServerErrorRate + annotations: + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-querier.*"}[5m])) + / + sum by (job) (rate(grpc_server_started_total{job=~"thanos-querier.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: warning +- alert: ThanosQuerierGrpcClientErrorRate + annotations: + message: Thanos Query {{$labels.job}} is failing to send {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job=~"thanos-querier.*"}[5m])) + / + sum by (job) (rate(grpc_client_started_total{job=~"thanos-querier.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: warning +- alert: ThanosQuerierHighDNSFailures + annotations: + message: Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries. + expr: | + ( + sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + / + sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + > 1 + ) + for: 15m + labels: + severity: warning +- alert: ThanosQuerierInstantLatencyHigh + annotations: + message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value + }} seconds for instant queries. + expr: | + ( + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"})) > 10 + and + sum by (job) (rate(http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"}[5m])) > 0 + ) + for: 10m + labels: + severity: critical +- alert: ThanosQuerierRangeLatencyHigh + annotations: + message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value + }} seconds for instant queries. + expr: | + ( + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query_range"})) > 10 + and + sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-querier.*", handler="query_range"}[5m])) > 0 + ) + for: 10m + labels: + severity: critical +``` + +## Receive + +[embedmd]:# (../tmp/thanos-receiver.rules.yaml yaml) +```yaml +name: thanos-receiver.rules +rules: +- alert: ThanosReceiverHttpRequestErrorRateHigh + annotations: + message: Thanos Receive {{$labels.job}} is failing to handle {{ $value | humanize + }}% of requests. + expr: | + ( + sum(rate(http_requests_total{code=~"5..", job=~"thanos-receiver.*", handler="receive"}[5m])) + / + sum(rate(http_requests_total{job=~"thanos-receiver.*", handler="receive"}[5m])) + ) * 100 > 5 + for: 5m + labels: + severity: critical +- alert: ThanosReceiverHttpRequestLatencyHigh + annotations: + message: Thanos Receive {{$labels.job}} has a 99th percentile latency of {{ $value + }} seconds for requests. + expr: | + ( + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-receiver.*", handler="receive"})) > 10 + and + sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-receiver.*", handler="receive"}[5m])) > 0 + ) + for: 10m + labels: + severity: critical +- alert: ThanosReceiverHighForwardRequestFailures + annotations: + message: Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receiver.*"}[5m])) + / + sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receiver.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: critical +- alert: ThanosReceiverHighHashringFileRefreshFailures + annotations: + message: Thanos Receive {{$labels.job}} is failing to refresh hashring file, {{ + $value | humanize }} of attempts failed. + expr: | + ( + sum by (job) (rate(thanos_receive_hashrings_file_errors_total{job=~"thanos-receiver.*"}[5m])) + / + sum by (job) (rate(thanos_receive_hashrings_file_refreshes_total{job=~"thanos-receiver.*"}[5m])) + > 0 + ) + for: 15m + labels: + severity: warning +- alert: ThanosReceiverConfigReloadFailure + annotations: + message: Thanos Receive {{$labels.job}} has not been able to reload hashring configurations. + expr: avg(thanos_receive_config_last_reload_successful{job=~"thanos-receiver.*"}) + by (job) != 1 + for: 5m + labels: + severity: warning ``` -- alert: ThanosQueryGrpcErrorRate - expr: rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable",name="prometheus"}[5m]) > 0 + +## Extras + +### Absent Rules + +[embedmd]:# (../tmp/thanos-component-absent.rules.yaml yaml) +```yaml +name: thanos-component-absent.rules +rules: +- alert: ThanosCompactorIsDown + annotations: + message: ThanosCompactor has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-compactor.*"} == 1) for: 5m labels: - team: TEAM + severity: critical +- alert: ThanosQuerierIsDown annotations: - summary: Thanos Query is returning Internal/Unavailable errors - impact: Grafana is not showing metrics - action: Check {{ $labels.kubernetes_pod_name }} pod logs in {{ $labels.kubernetes_namespace}} namespace - dashboard: QUERY_URL + message: ThanosQuerier has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-querier.*"} == 1) + for: 5m + labels: + severity: critical +- alert: ThanosReceiverIsDown + annotations: + message: ThanosReceiver has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-receiver.*"} == 1) + for: 5m + labels: + severity: critical +- alert: ThanosRulerIsDown + annotations: + message: ThanosRuler has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-ruler.*"} == 1) + for: 5m + labels: + severity: critical +- alert: ThanosSidecarIsDown + annotations: + message: ThanosSidecar has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-sidecar.*"} == 1) + for: 5m + labels: + severity: critical +- alert: ThanosStoreIsDown + annotations: + message: ThanosStore has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-store.*"} == 1) + for: 5m + labels: + severity: critical ``` diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml new file mode 100644 index 0000000000..23f10b655f --- /dev/null +++ b/examples/alerts/alerts.yaml @@ -0,0 +1,329 @@ +groups: +- name: thanos-compactor.rules + rules: + - alert: ThanosCompactorMultipleCompactsAreRunning + annotations: + message: You should never run more than one Thanos Compact at once. You have + {{ $value }} + expr: sum(up{job=~"thanos-compactor.*"}) > 1 + for: 5m + labels: + severity: warning + - alert: ThanosCompactorHalted + annotations: + message: Thanos Compact {{$labels.job}} has failed to run and now is halted. + expr: thanos_compactor_halted{job=~"thanos-compactor.*"} == 1 + for: 5m + labels: + severity: warning + - alert: ThanosCompactorHighCompactionFailures + annotations: + message: Thanos Compact {{$labels.job}} is failing to execute {{ $value | humanize + }}% of compactions. + expr: | + ( + sum by (job) (rate(thanos_compact_group_compactions_failures_total{job=~"thanos-compactor.*"}[5m])) + / + sum by (job) (rate(thanos_compact_group_compactions_total{job=~"thanos-compactor.*"}[5m])) + * 100 > 5 + ) + for: 15m + labels: + severity: warning + - alert: ThanosCompactorBucketHighOperationFailures + annotations: + message: Thanos Compact {{$labels.job}} Bucket is failing to execute {{ $value + | humanize }}% of operations. + expr: | + ( + sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-compactor.*"}[5m])) + / + sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~"thanos-compactor.*"}[5m])) + * 100 > 5 + ) + for: 15m + labels: + severity: warning + - alert: ThanosCompactorHasNotRun + annotations: + message: Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours. + expr: (time() - max(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compactor.*"})) + / 60 / 60 > 24 + labels: + severity: warning +- name: thanos-querier.rules + rules: + - alert: ThanosQuerierHttpRequestQueryErrorRateHigh + annotations: + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + }}% of "query" requests. + expr: | + ( + sum(rate(http_requests_total{code=~"5..", job=~"thanos-querier.*", handler="query"}[5m])) + / + sum(rate(http_requests_total{job=~"thanos-querier.*", handler="query"}[5m])) + ) * 100 > 5 + for: 5m + labels: + severity: critical + - alert: ThanosQuerierHttpRequestQueryRangeErrorRateHigh + annotations: + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + }}% of "query_range" requests. + expr: | + ( + sum(rate(http_requests_total{code=~"5..", job=~"thanos-querier.*", handler="query_range"}[5m])) + / + sum(rate(http_requests_total{job=~"thanos-querier.*", handler="query_range"}[5m])) + ) * 100 > 5 + for: 5m + labels: + severity: critical + - alert: ThanosQuerierGrpcServerErrorRate + annotations: + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-querier.*"}[5m])) + / + sum by (job) (rate(grpc_server_started_total{job=~"thanos-querier.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: warning + - alert: ThanosQuerierGrpcClientErrorRate + annotations: + message: Thanos Query {{$labels.job}} is failing to send {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job=~"thanos-querier.*"}[5m])) + / + sum by (job) (rate(grpc_client_started_total{job=~"thanos-querier.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: warning + - alert: ThanosQuerierHighDNSFailures + annotations: + message: Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries. + expr: | + ( + sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + / + sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + > 1 + ) + for: 15m + labels: + severity: warning + - alert: ThanosQuerierInstantLatencyHigh + annotations: + message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value + }} seconds for instant queries. + expr: | + ( + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"})) > 10 + and + sum by (job) (rate(http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"}[5m])) > 0 + ) + for: 10m + labels: + severity: critical + - alert: ThanosQuerierRangeLatencyHigh + annotations: + message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value + }} seconds for instant queries. + expr: | + ( + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query_range"})) > 10 + and + sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-querier.*", handler="query_range"}[5m])) > 0 + ) + for: 10m + labels: + severity: critical +- name: thanos-receiver.rules + rules: + - alert: ThanosReceiverHttpRequestErrorRateHigh + annotations: + message: Thanos Receive {{$labels.job}} is failing to handle {{ $value | humanize + }}% of requests. + expr: | + ( + sum(rate(http_requests_total{code=~"5..", job=~"thanos-receiver.*", handler="receive"}[5m])) + / + sum(rate(http_requests_total{job=~"thanos-receiver.*", handler="receive"}[5m])) + ) * 100 > 5 + for: 5m + labels: + severity: critical + - alert: ThanosReceiverHttpRequestLatencyHigh + annotations: + message: Thanos Receive {{$labels.job}} has a 99th percentile latency of {{ + $value }} seconds for requests. + expr: | + ( + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-receiver.*", handler="receive"})) > 10 + and + sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-receiver.*", handler="receive"}[5m])) > 0 + ) + for: 10m + labels: + severity: critical + - alert: ThanosReceiverHighForwardRequestFailures + annotations: + message: Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receiver.*"}[5m])) + / + sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receiver.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: critical + - alert: ThanosReceiverHighHashringFileRefreshFailures + annotations: + message: Thanos Receive {{$labels.job}} is failing to refresh hashring file, + {{ $value | humanize }} of attempts failed. + expr: | + ( + sum by (job) (rate(thanos_receive_hashrings_file_errors_total{job=~"thanos-receiver.*"}[5m])) + / + sum by (job) (rate(thanos_receive_hashrings_file_refreshes_total{job=~"thanos-receiver.*"}[5m])) + > 0 + ) + for: 15m + labels: + severity: warning + - alert: ThanosReceiverConfigReloadFailure + annotations: + message: Thanos Receive {{$labels.job}} has not been able to reload hashring + configurations. + expr: avg(thanos_receive_config_last_reload_successful{job=~"thanos-receiver.*"}) + by (job) != 1 + for: 5m + labels: + severity: warning +- name: thanos-sidecar.rules + rules: + - alert: ThanosSidecarUnhealthy + annotations: + message: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for {{ + $value }} seconds. + expr: | + count(time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"}) by (job, pod) >= 300) > 0 + labels: + severity: critical +- name: thanos-store.rules + rules: + - alert: ThanosStoreGrpcErrorRate + annotations: + message: Thanos Store {{$labels.job}} is failing to handle {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-store.*"}[5m])) + / + sum by (job) (rate(grpc_server_started_total{job=~"thanos-store.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: warning + - alert: ThanosStoreSeriesGateLatencyHigh + annotations: + message: Thanos Store {{$labels.job}} has a 99th percentile latency of {{ $value + }} seconds for store series gate requests. + expr: | + ( + histogram_quantile(0.9, sum by (job, le) (thanos_bucket_store_series_gate_duration_seconds_bucket{job=~"thanos-store.*"})) > 2 + and + sum by (job) (rate(thanos_bucket_store_series_gate_duration_seconds_count{job=~"thanos-store.*"}[5m])) > 0 + ) + for: 10m + labels: + severity: warning + - alert: ThanosStoreBucketHighOperationFailures + annotations: + message: Thanos Store {{$labels.job}} Bucket is failing to execute {{ $value + | humanize }}% of operations. + expr: | + ( + sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-store.*"}[5m])) + / + sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~"thanos-store.*"}[5m])) + * 100 > 5 + ) + for: 15m + labels: + severity: warning + - alert: ThanosStoreObjstoreOperationLatencyHigh + annotations: + message: Thanos Store {{$labels.job}} Bucket has a 99th percentile latency of + {{ $value }} seconds for the bucket operations. + expr: | + ( + histogram_quantile(0.9, sum by (job, le) (thanos_objstore_bucket_operation_duration_seconds_bucket{job=~"thanos-store.*"})) > 15 + and + sum by (job) (rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~"thanos-store.*"}[5m])) > 0 + ) + for: 10m + labels: + severity: warning +- name: thanos-component-absent.rules + rules: + - alert: ThanosCompactorIsDown + annotations: + message: ThanosCompactor has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-compactor.*"} == 1) + for: 5m + labels: + severity: critical + - alert: ThanosQuerierIsDown + annotations: + message: ThanosQuerier has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-querier.*"} == 1) + for: 5m + labels: + severity: critical + - alert: ThanosReceiverIsDown + annotations: + message: ThanosReceiver has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-receiver.*"} == 1) + for: 5m + labels: + severity: critical + - alert: ThanosRulerIsDown + annotations: + message: ThanosRuler has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-ruler.*"} == 1) + for: 5m + labels: + severity: critical + - alert: ThanosSidecarIsDown + annotations: + message: ThanosSidecar has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-sidecar.*"} == 1) + for: 5m + labels: + severity: critical + - alert: ThanosStoreIsDown + annotations: + message: ThanosStore has disappeared from Prometheus target discovery. + expr: | + absent(up{job=~"thanos-store.*"} == 1) + for: 5m + labels: + severity: critical diff --git a/examples/alerts/rules.yaml b/examples/alerts/rules.yaml new file mode 100644 index 0000000000..82190e63c6 --- /dev/null +++ b/examples/alerts/rules.yaml @@ -0,0 +1,123 @@ +groups: +- name: thanos-querier.rules + rules: + - expr: | + ( + sum(rate(grpc_client_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-querier.*", grpc_type="unary"}[5m])) + / + sum(rate(grpc_client_started_total{job=~"thanos-querier.*", grpc_type="unary"}[5m])) + ) + labels: {} + record: :grpc_client_failures_per_unary:sum_rate + - expr: | + ( + sum(rate(grpc_client_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-querier.*", grpc_type="server_stream"}[5m])) + / + sum(rate(grpc_client_started_total{job=~"thanos-querier.*", grpc_type="server_stream"}[5m])) + ) + labels: {} + record: :grpc_client_failures_per_stream:sum_rate + - expr: | + ( + sum(rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + / + sum(rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + ) + labels: {} + record: :thanos_querier_store_apis_dns_failures_per_lookup:sum_rate + - expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"}[5m])) by (le) + ) + labels: + quantile: "0.99" + record: :query_duration_seconds:histogram_quantile + - expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query_range"}[5m])) by (le) + ) + labels: + quantile: "0.99" + record: :api_range_query_duration_seconds:histogram_quantile +- name: thanos-receiver.rules + rules: + - expr: | + sum( + rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-receiver.*", grpc_type="unary"}[5m]) + / + rate(grpc_server_started_total{job=~"thanos-receiver.*", grpc_type="unary"}[5m]) + ) + labels: {} + record: :grpc_server_failures_per_unary:sum_rate + - expr: | + sum( + rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-receiver.*", grpc_type="server_stream"}[5m]) + / + rate(grpc_server_started_total{job=~"thanos-receiver.*", grpc_type="server_stream"}[5m]) + ) + labels: {} + record: :grpc_server_failures_per_stream:sum_rate + - expr: | + sum( + rate(http_requests_total{handler="receive", job=~"thanos-receiver.*", code!~"5.."}[5m]) + / + rate(http_requests_total{handler="receive", job=~"thanos-receiver.*"}[5m]) + ) + labels: {} + record: :http_failure_per_request:sum_rate + - expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{handler="receive", job=~"thanos-receiver.*"}[5m])) by (le) + ) + labels: + quantile: "0.99" + record: :http_request_duration_seconds:histogram_quantile + - expr: | + ( + sum(rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receiver.*"}[5m])) + / + sum(rate(thanos_receive_forward_requests_total{job=~"thanos-receiver.*"}[5m])) + ) + labels: {} + record: :thanos_receive_forward_failure_per_requests:sum_rate + - expr: | + ( + sum(rate(thanos_receive_hashrings_file_errors_total{job=~"thanos-receiver.*"}[5m])) + / + sum(rate(thanos_receive_hashrings_file_refreshes_total{job=~"thanos-receiver.*"}[5m])) + ) + labels: {} + record: :thanos_receive_hashring_file_failure_per_refresh:sum_rate +- name: thanos-store.rules + rules: + - expr: | + ( + sum(rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-store.*", grpc_type="unary"}[5m])) + / + sum(rate(grpc_server_started_total{job=~"thanos-store.*", grpc_type="unary"}[5m])) + ) + labels: {} + record: :grpc_server_failures_per_unary:sum_rate + - expr: | + ( + sum(rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-store.*", grpc_type="server_stream"}[5m])) + / + sum(rate(grpc_server_started_total{job=~"thanos-store.*", grpc_type="server_stream"}[5m])) + ) + labels: {} + record: :grpc_server_failures_per_stream:sum_rate + - expr: | + ( + sum(rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-store.*"}[5m])) + / + sum(rate(thanos_objstore_bucket_operations_total{job=~"thanos-store.*"}[5m])) + ) + labels: {} + record: :thanos_objstore_bucket_failures_per_operation:sum_rate + - expr: | + histogram_quantile(0.99, + sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~"thanos-store.*"}[5m])) by (le) + ) + labels: + quantile: "0.99" + record: :thanos_objstore_bucket_operation_duration_seconds:histogram_quantile diff --git a/examples/alerts/tests.yaml b/examples/alerts/tests.yaml new file mode 100644 index 0000000000..25df0414e4 --- /dev/null +++ b/examples/alerts/tests.yaml @@ -0,0 +1,94 @@ +rule_files: + - alerts.yaml + - rules.yaml + +evaluation_interval: 1m + +tests: +- interval: 1m + input_series: + - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", pod="thanos-sidecar-pod-0"}' + values: '5 10 43 17 11 0 0 0' + - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", pod="thanos-sidecar-pod-1"}' + values: '4 9 42 15 10 0 0 0' + promql_expr_test: + - expr: time() + eval_time: 1m + exp_samples: + - labels: '{}' + value: 60 + - expr: time() + eval_time: 2m + exp_samples: + - labels: '{}' + value: 120 + - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (pod) + eval_time: 2m + exp_samples: + - labels: '{pod="thanos-sidecar-pod-0"}' + value: 43 + - labels: '{pod="thanos-sidecar-pod-1"}' + value: 42 + - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (pod) + eval_time: 5m + exp_samples: + - labels: '{pod="thanos-sidecar-pod-0"}' + value: 0 + - labels: '{pod="thanos-sidecar-pod-1"}' + value: 0 + - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (pod) + eval_time: 6m + exp_samples: + - labels: '{pod="thanos-sidecar-pod-0"}' + value: 0 + - labels: '{pod="thanos-sidecar-pod-1"}' + value: 0 + - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (pod) + eval_time: 5m + exp_samples: + - labels: '{pod="thanos-sidecar-pod-0"}' + value: 300 + - labels: '{pod="thanos-sidecar-pod-1"}' + value: 300 + - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (pod) + eval_time: 6m + exp_samples: + - labels: '{pod="thanos-sidecar-pod-0"}' + value: 360 + - labels: '{pod="thanos-sidecar-pod-1"}' + value: 360 + - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (pod) >= 300 + eval_time: 12m + exp_samples: + - labels: '{pod="thanos-sidecar-pod-0"}' + value: 720 + - labels: '{pod="thanos-sidecar-pod-1"}' + value: 720 + alert_rule_test: + - eval_time: 1m + alertname: ThanosSidecarUnhealthy + - eval_time: 2m + alertname: ThanosSidecarUnhealthy + - eval_time: 3m + alertname: ThanosSidecarUnhealthy + - eval_time: 5m + alertname: ThanosSidecarUnhealthy + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + message: 'Thanos Sidecar is unhealthy for 2 seconds.' + - eval_time: 6m + alertname: ThanosSidecarUnhealthy + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + message: 'Thanos Sidecar is unhealthy for 2 seconds.' + - eval_time: 12m + alertname: ThanosSidecarUnhealthy + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + message: 'Thanos Sidecar is unhealthy for 2 seconds.' diff --git a/examples/dashboards/compactor.json b/examples/dashboards/compactor.json new file mode 100644 index 0000000000..9e2d3d65cc --- /dev/null +++ b/examples/dashboards/compactor.json @@ -0,0 +1,1549 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of execution for compactions against blocks that are stored in the bucket by compaction group.", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_group_compactions_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, group)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "compaction {{job}} {{group}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of executed compactions against blocks that are stored in the bucket.", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_group_compactions_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_compact_group_compactions_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Group Compaction", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of execution for downsampling against blocks that are stored in the bucket by compaction group.", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_downsample_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, group)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "downsample {{job}} {{group}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of executed downsampling against blocks that are stored in the bucket.", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_downsample_failed_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_compact_downsample_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Downsample", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of execution for removals of blocks if their data is available as part of a block with a higher compaction level.", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_garbage_collection_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "garbage collection {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of executed garbage collections.", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_garbage_collection_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_compact_garbage_collection_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to execute garbage collection in quantiles.", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_compact_garbage_collection_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(thanos_compact_garbage_collection_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_compact_garbage_collection_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(thanos_compact_garbage_collection_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Garbage Collection", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of execution for all meta files from blocks in the bucket into the memory.", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_sync_meta_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "sync {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of executed meta file sync.", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_sync_meta_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_compact_sync_meta_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to execute meta file sync, in quantiles.", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_compact_sync_meta_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(thanos_compact_sync_meta_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_compact_sync_meta_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(thanos_compact_sync_meta_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Sync Meta", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of execution for operations against the bucket.", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_objstore_bucket_operations_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, operation)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{operation}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of executed operations against the bucket.", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_objstore_bucket_operations_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to execute operations against the bucket, in quantiles.", + "fill": 1, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Object Store Operations", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_alloc_bytes_total{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_stack_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse stack {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse heap {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Used", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{namespace=\"$namespace\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_gc_duration_seconds{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC Time Quantiles", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Resources", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "thanos-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "thanos-compactor.*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": false, + "name": "job", + "options": [ ], + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-compactor.*\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ ], + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-compactor.*\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 300, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "query": "5m,10m,30m,1h,6h,12h", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Thanos / Compactor", + "uid": "8378cc7f803b0bfae5809a101991ed76", + "version": 0 +} diff --git a/examples/dashboards/dashboards.md b/examples/dashboards/dashboards.md new file mode 100644 index 0000000000..36f13fe703 --- /dev/null +++ b/examples/dashboards/dashboards.md @@ -0,0 +1,18 @@ +# Dashboards + +There exists Grafana dashboards for each component (not all of them complete) targeted for environments running Kubernetes: + +- [Thanos Overview](thanos-overview.json) +- [Thanos Compact](thanos-compactor.json) +- [Thanos Querier](thanos-querier.json) +- [Thanos Store](thanos-store.json) +- [Thanos Receiver](thanos-receiver.json) +- [Thanos Sidecar](thanos-sidecar.json) +- [Thanos Ruler](thanos-ruler.json) + +You can import them via `Import -> Paste JSON` in Grafana. +These dashboards require Grafana 5 or above, importing them in older versions are known not to work. + +## Configuration + +All dashboards are generated using [`thanos-mixin`](../../jsonnet/thanos-mixin) and check out [README](../../jsonnet/thanos-mixin/README.md) for further information. diff --git a/examples/dashboards/overview.json b/examples/dashboards/overview.json new file mode 100644 index 0000000000..f328a3e370 --- /dev/null +++ b/examples/dashboards/overview.json @@ -0,0 +1,2066 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of requests against /query for the given time.", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Querier", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Querier", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(label_replace(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Requests Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the the total number of handled requests against /query.", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Querier", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Querier", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Requests Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests.", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "dashboard": "Thanos / Querier", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Querier", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query\"}[$interval])) by (job, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} P99", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 0.5, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency 99th Percentile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Instant Query", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of requests against /query_range for the given time range.", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Querier", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Querier", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(label_replace(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query_range\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Requests Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the the total number of handled requests against /query_range.", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Querier", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Querier", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query_range\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query_range\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Requests Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests.", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "dashboard": "Thanos / Querier", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Querier", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query_range\"}[$interval])) by (job, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} P99", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 0.5, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency 99th Percentile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Range Query", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests from queriers.", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Store", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Store", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"thanos-store.*\",grpc_type=\"unary\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "gPRC (Unary) Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Store", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Store", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"thanos-store.*\",grpc_type=\"unary\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"thanos-store.*\",grpc_type=\"unary\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "gPRC (Unary) Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers.", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "dashboard": "Thanos / Store", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Store", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type=\"unary\",namespace=\"$namespace\",job=~\"thanos-store.*\"}[$interval])) by (job, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} P99", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 0.5, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "gRPC Latency 99th Percentile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Store", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests from queriers.", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Sidecar", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Sidecar", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"thanos-sidecar.*\",grpc_type=\"unary\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "gPRC (Unary) Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Sidecar", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Sidecar", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"thanos-sidecar.*\",grpc_type=\"unary\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"thanos-sidecar.*\",grpc_type=\"unary\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "gPRC (Unary) Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "dashboard": "Thanos / Sidecar", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Sidecar", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type=\"unary\",namespace=\"$namespace\",job=~\"thanos-sidecar.*\"}[$interval])) by (job, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} P99", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 0.5, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "gPRC (Unary) Latency 99th Percentile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Sidecar", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of incoming requests.", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Receiver", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Receiver", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(label_replace(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receiver.*\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Requests Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled incoming requests.", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Receiver", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Receiver", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receiver.*\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receiver.*\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Requests Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle incoming requests.", + "fill": 1, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "dashboard": "Thanos / Receiver", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Receiver", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receiver.*\"}[$interval])) by (job, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} P99", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 0.5, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Requests Latency 99th Percentile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Receive", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of alerts that successfully sent to alert manager.", + "fill": 10, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Ruler", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Ruler", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}[$interval])) by (job, alertmanager)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{alertmanager}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Alert Sent Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of sent alerts.", + "fill": 10, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Ruler", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Ruler", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_alert_sender_errors_total{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}[$interval])) / sum(rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Alert Sent Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to send alerts to alert manager.", + "fill": 1, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "dashboard": "Thanos / Ruler", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Ruler", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_alert_sender_latency_seconds_bucket{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}[$interval])) by (job, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} P99", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 0.5, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Alert Sent Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rule", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of execution for compactions against blocks that are stored in the bucket by compaction group.", + "fill": 10, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Compactor", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Compactor", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_group_compactions_total{namespace=\"$namespace\",job=~\"thanos-compactor.*\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "compaction {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Compaction Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of executed compactions against blocks that are stored in the bucket.", + "fill": 10, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + { + "dashboard": "Thanos / Compactor", + "includeVars": true, + "keepTime": true, + "title": "Thanos / Compactor", + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_compact_group_compactions_failures_total{namespace=\"$namespace\",job=~\"thanos-compactor.*\"}[$interval])) / sum(rate(thanos_compact_group_compactions_total{namespace=\"$namespace\",job=~\"thanos-compactor.*\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Compaction Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Compact", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "thanos-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 300, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "query": "5m,10m,30m,1h,6h,12h", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Thanos / Overview", + "uid": "0cb8830a6e957978796729870f560cda", + "version": 0 +} diff --git a/examples/dashboards/querier.json b/examples/dashboards/querier.json new file mode 100644 index 0000000000..f69e569ab8 --- /dev/null +++ b/examples/dashboards/querier.json @@ -0,0 +1,2481 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of requests against /query for the given time.", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(label_replace(rate(http_requests_total{namespace=\"$namespace\",job=~\"$job\",handler=\"query\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the the total number of handled requests against /query.", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"$job\",handler=\"query\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"$job\",handler=\"query\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests in quantiles.", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",handler=\"query\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(http_request_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\",handler=\"query\"}[$interval])) by (job) * 1 / sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\",handler=\"query\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",handler=\"query\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Instant Query API", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of requests against /query_range for the given time range.", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(label_replace(rate(http_requests_total{namespace=\"$namespace\",job=~\"$job\",handler=\"query_range\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the the total number of handled requests against /query_range.", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"$job\",handler=\"query_range\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"$job\",handler=\"query_range\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests in quantiles.", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",handler=\"query_range\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(http_request_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\",handler=\"query_range\"}[$interval])) by (job) * 1 / sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\",handler=\"query_range\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",handler=\"query_range\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Range Query API", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of requests against /query for the given time, with handlers and codes.", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(label_replace(rate(http_requests_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, handler, status_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{handler}} {{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the the total number of handled requests, in more detail.", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"$job\",code!~\"2..\"}[$interval])) by (job, handler, code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{handler}} {{code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests in quantiles.", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, handler, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{handler}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(http_request_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, handler) * 1 / sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, handler)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{handler}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, handler, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{handler}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests from other queriers.", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the the total number of handled requests from other queriers.", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) / sum(rate(grpc_client_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from other queriers, in quantiles.", + "fill": 1, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_client_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_client_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Unary)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests, with grpc methods and codes from other queriers.", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the the total number of handled requests from other queriers.", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from other queriers, in quantiles.", + "fill": 1, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_client_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_client_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests from other queriers.", + "fill": 10, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the the total number of handled requests from other queriers.", + "fill": 10, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) / sum(rate(grpc_client_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from other queriers, in quantiles", + "fill": 1, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_client_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_client_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Stream)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests, with grpc methods and codes.", + "fill": 10, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the the total number of handled requests.", + "fill": 10, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests in quantiles.", + "fill": 1, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_client_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_client_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of DNS lookups to discover stores.", + "fill": 1, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_querier_store_apis_dns_lookups_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "lookups {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of failures compared to the the total number of executed DNS lookups.", + "fill": 10, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_querier_store_apis_dns_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_querier_store_apis_dns_lookups_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "DNS", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_alloc_bytes_total{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_stack_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse stack {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse heap {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Used", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{namespace=\"$namespace\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_gc_duration_seconds{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC Time Quantiles", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Resources", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "thanos-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "thanos-querier.*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": false, + "name": "job", + "options": [ ], + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-querier.*\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ ], + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-querier.*\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 300, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "query": "5m,10m,30m,1h,6h,12h", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Thanos / Querier", + "uid": "98fde97ddeaf2981041745f1f2ba68c2", + "version": 0 +} diff --git a/examples/dashboards/receiver.json b/examples/dashboards/receiver.json new file mode 100644 index 0000000000..3abba13206 --- /dev/null +++ b/examples/dashboards/receiver.json @@ -0,0 +1,2336 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of incoming requests.", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(label_replace(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled incoming requests.", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle incoming requests in quantiles.", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(http_request_duration_seconds_sum{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(http_request_duration_seconds_count{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Incoming Request", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of incoming requests.", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(label_replace(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, handler, status_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{handler}} {{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled incoming requests.", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\",code!~\"2..\"}[$interval])) by (job, handler, code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{handler}} {{code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle incoming requests in quantiles.", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, handler, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{handler}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(http_request_duration_seconds_sum{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, handler) * 1 / sum(rate(http_request_duration_seconds_count{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, handler)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{handler}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{handler=\"receive\",namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, handler, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{handler}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of forwarded requests to other receive nodes.", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_receive_forward_requests_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "all {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of forwareded requests to other receive nodes.", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_receive_forward_requests_total{namespace=\"$namespace\",job=~\"$job\",result=\"error\"}[$interval])) / sum(rate(thanos_receive_forward_requests_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Forward Request", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests from queriers.", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Unary)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests from queriers.", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests from queriers.", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Stream)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests from queriers.", + "fill": 10, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows the relative time of last successful upload to the object-store bucket.", + "fill": 1, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Uploaded Ago", + "colorMode": null, + "colors": [ ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value", + "thresholds": [ ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "time() - max(thanos_objstore_bucket_last_successful_upload_time{namespace=\"$namespace\",job=~\"$job\"}) by (job, bucket)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Successful Upload", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Last Updated", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_alloc_bytes_total{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_stack_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse stack {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse heap {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Used", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{namespace=\"$namespace\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_gc_duration_seconds{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC Time Quantiles", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Resources", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "thanos-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "thanos-receiver.*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": false, + "name": "job", + "options": [ ], + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-receiver.*\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ ], + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-receiver.*\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 300, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "query": "5m,10m,30m,1h,6h,12h", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Thanos / Receiver", + "uid": "b5958da86b143e45752506d3c09c4f92", + "version": 0 +} diff --git a/examples/dashboards/ruler.json b/examples/dashboards/ruler.json new file mode 100644 index 0000000000..fa2596da4a --- /dev/null +++ b/examples/dashboards/ruler.json @@ -0,0 +1,1846 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of dropped alerts.", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_alert_sender_alerts_dropped_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, alertmanager)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{alertmanager}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of alerts that successfully sent to alert manager.", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, alertmanager)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{alertmanager}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Sent Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of sent alerts.", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_alert_sender_errors_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Sent Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to send alerts to alert manager.", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_alert_sender_latency_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(thanos_alert_sender_latency_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_alert_sender_latency_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(thanos_alert_sender_latency_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Sent Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alert Sent", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests.", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests.", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests, in quantiles.", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Unary)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests.", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests.", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests, in quantiles.", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests.", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests.", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests, in quantiles", + "fill": 1, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Stream)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests.", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests.", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests, in quantiles", + "fill": 1, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_alloc_bytes_total{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_stack_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse stack {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse heap {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Used", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{namespace=\"$namespace\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_gc_duration_seconds{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC Time Quantiles", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Resources", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "thanos-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "thanos-ruler.*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": false, + "name": "job", + "options": [ ], + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ ], + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-ruler.*\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 300, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "query": "5m,10m,30m,1h,6h,12h", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Thanos / Ruler", + "uid": "ecc62fafbb8ae0213cb9188ec4f3b553", + "version": 0 +} diff --git a/examples/dashboards/sidecar.json b/examples/dashboards/sidecar.json new file mode 100644 index 0000000000..f0d5bbb432 --- /dev/null +++ b/examples/dashboards/sidecar.json @@ -0,0 +1,1889 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests from queriers.", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Unary)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests from queriers.", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests from queriers.", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Stream)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests from queriers.", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_client_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_client_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows the relative time of last successful upload to the object-store bucket.", + "fill": 1, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Uploaded Ago", + "colorMode": null, + "colors": [ ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value", + "thresholds": [ ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "time() - max(thanos_objstore_bucket_last_successful_upload_time{namespace=\"$namespace\",job=~\"$job\"}) by (job, bucket)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Successful Upload", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Last Updated", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_objstore_bucket_operations_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, operation)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{operation}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_objstore_bucket_operations_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bucket Operations", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_alloc_bytes_total{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_stack_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse stack {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse heap {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Used", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{namespace=\"$namespace\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_gc_duration_seconds{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC Time Quantiles", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Resources", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "thanos-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "thanos-sidecar.*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": false, + "name": "job", + "options": [ ], + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-sidecar.*\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ ], + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-sidecar.*\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 300, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "query": "5m,10m,30m,1h,6h,12h", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Thanos / Sidecar", + "uid": "b19644bfbf0ec1e108027cce268d99f7", + "version": 0 +} diff --git a/examples/dashboards/store.json b/examples/dashboards/store.json new file mode 100644 index 0000000000..4b0f0aaba9 --- /dev/null +++ b/examples/dashboards/store.json @@ -0,0 +1,3115 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests from queriers.", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Unary)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Unary gRPC requests from queriers.", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"unary\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Aborted": "#EAB839", + "AlreadyExists": "#7EB26D", + "Canceled": "#E24D42", + "DataLoss": "#E24D42", + "DeadlineExceeded": "#E24D42", + "FailedPrecondition": "#6ED0E0", + "Internal": "#E24D42", + "InvalidArgument": "#EF843C", + "NotFound": "#EF843C", + "OK": "#7EB26D", + "OutOfRange": "#E24D42", + "PermissionDenied": "#EF843C", + "ResourceExhausted": "#E24D42", + "Unauthenticated": "#EF843C", + "Unavailable": "#E24D42", + "Unimplemented": "#6ED0E0", + "Unknown": "#E24D42", + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests from queriers.", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) / sum(rate(grpc_server_started_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "gRPC (Stream)", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of handled Streamed gRPC requests from queriers.", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_client_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", + "fill": 1, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(grpc_client_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_client_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}} {{grpc_method}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Detailed", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of execution for operations against the bucket.", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_objstore_bucket_operations_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, operation)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{operation}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of executed operations against the bucket.", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_objstore_bucket_operations_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to execute operations against the bucket, in quantiles.", + "fill": 1, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bucket Operations", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of block loads from the bucket.", + "fill": 10, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_bucket_store_block_loads_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block loads", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block Load Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of block loads from the bucket.", + "fill": 10, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_bucket_store_block_load_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_bucket_store_block_loads_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block Load Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of block drops.", + "fill": 10, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_bucket_store_block_drops_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, operation)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block drops {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block Drop Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of block drops.", + "fill": 10, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_bucket_store_block_drop_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_bucket_store_block_drops_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block Drop Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Block Operations", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Show rate of cache requests.", + "fill": 10, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_store_index_cache_requests_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, item_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{item_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of errors compared to the total number of cache hits.", + "fill": 10, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_store_index_cache_hits_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, item_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{item_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Hits", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Show rate of added items to cache.", + "fill": 10, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_store_index_cache_items_added_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, item_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{item_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Added", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Show rate of evicted items from cache.", + "fill": 10, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_store_index_cache_items_evicted_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, item_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{item_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Evicted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Cache Operations", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows size of chunks that have sent to the bucket, in bytes.", + "fill": 1, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_bucket_store_sent_chunk_size_bytes_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(thanos_bucket_store_sent_chunk_size_bytes_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) / sum(rate(thanos_bucket_store_sent_chunk_size_bytes_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean", + "legendLink": null, + "step": 10 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_bucket_store_sent_chunk_size_bytes_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Chunk Size", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Store Sent", + "titleSize": "h6", + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "thanos_bucket_store_series_blocks_queried{namespace=\"$namespace\",job=~\"$job\",quantile=\"0.99\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(thanos_bucket_store_series_blocks_queried_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) / sum(rate(thanos_bucket_store_series_blocks_queried_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "thanos_bucket_store_series_blocks_queried{namespace=\"$namespace\",job=~\"$job\",quantile=\"0.50\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block queried", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "thanos_bucket_store_series_data_fetched{namespace=\"$namespace\",job=~\"$job\",quantile=\"0.99\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(thanos_bucket_store_series_data_fetched_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) / sum(rate(thanos_bucket_store_series_data_fetched_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "thanos_bucket_store_series_data_fetched{namespace=\"$namespace\",job=~\"$job\",quantile=\"0.50\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Data Fetched", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "thanos_bucket_store_series_result_series{namespace=\"$namespace\",job=~\"$job\",quantile=\"0.99\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(rate(thanos_bucket_store_series_result_series_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) / sum(rate(thanos_bucket_store_series_result_series_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "thanos_bucket_store_series_result_series{namespace=\"$namespace\",job=~\"$job\",quantile=\"0.50\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Result series", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Series Operations", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to get all series.", + "fill": 1, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_bucket_store_series_get_all_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(thanos_bucket_store_series_get_all_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_bucket_store_series_get_all_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(thanos_bucket_store_series_get_all_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Get All", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken to merge series.", + "fill": 1, + "id": 29, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Merge", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows how long has it taken for a series to wait at the gate.", + "fill": 1, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P99 {{job}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mean {{job}}", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "P50 {{job}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Gate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Series Operation Durations", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_alloc_bytes_total{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate all {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "rate(go_memstats_heap_alloc_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc rate heap {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_stack_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse stack {{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inuse heap {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Used", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{namespace=\"$namespace\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_gc_duration_seconds{namespace=\"$namespace\",job=~\"$job\",kubernetes_pod_name=~\"$pod\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC Time Quantiles", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Resources", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "thanos-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "thanos-store.*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": false, + "name": "job", + "options": [ ], + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-store.*\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ ], + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-store.*\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 300, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "query": "5m,10m,30m,1h,6h,12h", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Thanos / Store", + "uid": "e832e8f26403d95fac0ea1c59837588b", + "version": 0 +} diff --git a/examples/grafana/monitoring.md b/examples/grafana/monitoring.md deleted file mode 100644 index 8127f327e4..0000000000 --- a/examples/grafana/monitoring.md +++ /dev/null @@ -1,42 +0,0 @@ -# Grafana Dashboards - -There are 4 Grafana dashboards targeted for environments running Kubernetes: - -- [Thanos Compact](thanos-compact.json) -- [Thanos Query](thanos-query.json) -- [Thanos Store](thanos-store.json) -- [Thanos Sidecar](thanos-sidecar.json) -- [Thanos Rule](thanos-rule.json) - -You can import them via `Import -> Paste JSON` in Grafana. -These dashboards require Grafana 5, importing them in older versions are known not to work. - -# Configuration - -All dashboards can be configured via `labelselector` and `labelvalue` constants, which are used to pinpoint Thanos components. - -Let's say we have a service configured with following annotation: - -``` -apiVersion: v1 -kind: Service -metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10902" - prometheus.io/scrape: "true" - labels: - name: prometheus - name: prometheus -spec: - ports: - - name: prometheus - port: 9090 - protocol: TCP - targetPort: 9090 - selector: - app: prometheus -``` - -In this case `labelselector` should be `name` and `labelvalue` should be `prometheus` as metrics will have `name="prometheus"` label associated with them. - diff --git a/examples/grafana/thanos-compact.json b/examples/grafana/thanos-compact.json deleted file mode 100644 index ff54cacac3..0000000000 --- a/examples/grafana/thanos-compact.json +++ /dev/null @@ -1,856 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - }, - { - "name": "VAR_LABELSELECTOR", - "type": "constant", - "label": "labelselector", - "value": "app", - "description": "" - }, - { - "name": "VAR_LABELVALUE", - "type": "constant", - "label": "labelvalue", - "value": "thanos-compact", - "description": "" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1529906222108, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 22, - "panels": [], - "repeat": null, - "title": "Thanos Compact", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(prometheus_tsdb_compactions_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "compaction {{kubernetes_namespace}}", - "refId": "B" - }, - { - "expr": "sum(rate(thanos_objstore_bucket_operations_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "bucket ops {{kubernetes_namespace}}", - "refId": "A" - }, - { - "expr": "sum(rate(thanos_compact_garbage_collection_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "gc ops {{kubernetes_namespace}}", - "refId": "C" - }, - { - "expr": "sum(rate(thanos_compact_group_compactions_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "group compact {{kubernetes_namespace}}", - "refId": "D" - }, - { - "expr": "sum(rate(thanos_compact_sync_meta_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "sync metas {{kubernetes_namespace}}", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Operations/s [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "avg", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(prometheus_tsdb_compactions_failed_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "compaction {{kubernetes_namespace}}", - "refId": "B" - }, - { - "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "bucket ops {{kubernetes_namespace}}", - "refId": "A" - }, - { - "expr": "sum(rate(thanos_compact_garbage_collection_failures_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "gc ops {{kubernetes_namespace}}", - "refId": "C" - }, - { - "expr": "sum(rate(thanos_compact_group_compactions_failures_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "group compact {{kubernetes_namespace}}", - "refId": "D" - }, - { - "expr": "sum(rate(thanos_compact_sync_meta_failures_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "sync metas {{kubernetes_namespace}}", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Operation Failures/s [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(thanos_compact_garbage_collection_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,le))", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "99.99 gc {{kubernetes_namespace}}", - "refId": "A", - "step": 2 - }, - { - "expr": "histogram_quantile(0.9999, sum(rate(thanos_compact_sync_meta_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99.99 sync meta {{kubernetes_namespace}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.9999, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,le))", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "99.99 bucket ops {{kubernetes_namespace}}", - "refId": "C" - }, - { - "expr": "histogram_quantile(0.9999, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.99 compact {{kubernetes_namespace}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Operation Time Quantile [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 25, - "panels": [], - "repeat": null, - "title": "Ops", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_memstats_heap_alloc_bytes{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory Used", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_gc_duration_seconds{$labelselector=\"$labelvalue\", quantile=\"1\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}} ", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC Time Quantiles", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "refresh": "30s", - "revision": null, - "schemaVersion": 16, - "style": "dark", - "tags": [ - "thanos" - ], - "templating": { - "list": [ - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "1m", - "value": "1m" - }, - "hide": 0, - "label": null, - "name": "interval", - "options": [ - { - "selected": true, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "type": "interval" - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": true, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": "label_values(kubernetes_namespace)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - }, - "hide": 2, - "label": null, - "name": "labelselector", - "options": [ - { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - } - ], - "query": "${VAR_LABELSELECTOR}", - "type": "constant" - }, - { - "current": { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - }, - "hide": 2, - "label": null, - "name": "labelvalue", - "options": [ - { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - } - ], - "query": "${VAR_LABELVALUE}", - "type": "constant" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Thanos Compaction", - "uid": "s48S7j4ik", - "version": 3 -} diff --git a/examples/grafana/thanos-query.json b/examples/grafana/thanos-query.json deleted file mode 100644 index 424437e554..0000000000 --- a/examples/grafana/thanos-query.json +++ /dev/null @@ -1,1038 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - }, - { - "name": "VAR_LABELVALUE", - "type": "constant", - "label": "labelvalue", - "value": "thanos-query", - "description": "" - }, - { - "name": "VAR_LABELSELECTOR", - "type": "constant", - "label": "labelselector", - "value": "app", - "description": "" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1529906270695, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 22, - "panels": [], - "repeat": null, - "title": "Thanos Query", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(grpc_client_handled_total{$labelselector=\"$labelvalue\",kubernetes_pod_name=~\"$pod\"}[$interval])) by (kubernetes_pod_name, grpc_code, grpc_method)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{grpc_code}} {{grpc_method}} {{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Request RPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 27, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "avg", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(grpc_client_handling_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_pod_name=~\"$pod\"}[$interval])) by (grpc_method,kubernetes_pod_name, le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.99 {{grpc_method}} {{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Response Time Quantile [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 32, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "avg", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(http_request_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_pod_name=~\"$pod\",handler=\"query\"}[$interval])) by (kubernetes_pod_name, le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "instant_query {{kubernetes_pod_name}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.9999, sum(rate(http_request_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_pod_name=~\"$pod\",handler=\"query_range\"}[$interval])) by (kubernetes_pod_name, le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "range_query {{kubernetes_pod_name}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Thanos Query 99.99 Quantile [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_engine_query_duration_seconds{$labelselector=\"$labelvalue\",kubernetes_pod_name=~\"$pod\",quantile=\"0.99\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}} {{slice}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Prometheus Query 99 Quantile", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 29, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_engine_queries{$labelselector=\"$labelvalue\",kubernetes_pod_name=~\"$pod\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Prometheus Queries/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "columns": [], - "datasource": "${DS_PROMETHEUS}", - "fontSize": "100%", - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 29 - }, - "hideTimeOverride": false, - "id": 34, - "links": [], - "pageSize": null, - "scroll": false, - "showHeader": true, - "sort": { - "col": 1, - "desc": false - }, - "styles": [ - { - "alias": "Peer", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "external_labels", - "thresholds": [], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Time", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "Replicas", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Value", - "thresholds": [], - "type": "hidden", - "unit": "short" - } - ], - "targets": [ - { - "expr": "min(thanos_store_node_info{$labelselector=\"$labelvalue\"}) by (external_labels)", - "format": "table", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "title": "Gossip info", - "transform": "table", - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 25, - "panels": [], - "repeat": null, - "title": "Ops", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_memstats_heap_alloc_bytes{$labelselector=\"$labelvalue\",kubernetes_pod_name=~\"$pod\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory Used", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{$labelselector=\"$labelvalue\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 46 - }, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_gc_duration_seconds{$labelselector=\"$labelvalue\",kubernetes_pod_name=~\"$pod\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{quantile}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC Time Quantiles", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "refresh": "30s", - "revision": null, - "schemaVersion": 16, - "style": "dark", - "tags": [ - "thanos" - ], - "templating": { - "list": [ - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "1m", - "value": "1m" - }, - "hide": 0, - "label": null, - "name": "interval", - "options": [ - { - "selected": true, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "type": "interval" - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": true, - "label": "pod", - "multi": false, - "name": "pod", - "options": [], - "query": "label_values(thanos_build_info{$labelselector=\"$labelvalue\"}, kubernetes_pod_name)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - }, - "hide": 2, - "label": null, - "name": "labelvalue", - "options": [ - { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - } - ], - "query": "${VAR_LABELVALUE}", - "type": "constant" - }, - { - "current": { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - }, - "hide": 2, - "label": null, - "name": "labelselector", - "options": [ - { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - } - ], - "query": "${VAR_LABELSELECTOR}", - "type": "constant" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Thanos Query", - "uid": "opwl5gSiz", - "version": 2 -} diff --git a/examples/grafana/thanos-rule.json b/examples/grafana/thanos-rule.json deleted file mode 100644 index 3ac26f0530..0000000000 --- a/examples/grafana/thanos-rule.json +++ /dev/null @@ -1,1205 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - }, - { - "name": "VAR_LABELSELECTOR", - "type": "constant", - "label": "labelselector", - "value": "app", - "description": "" - }, - { - "name": "VAR_LABELVALUE", - "type": "constant", - "label": "labelvalue", - "value": "thanos-rule", - "description": "" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.4" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1530195209483, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 22, - "panels": [], - "repeat": null, - "title": "Thanos Rule", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 29, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(grpc_server_handled_total{kubernetes_namespace=~\"$namespace\",$labelselector=\"$labelvalue\"}[$interval])) by (grpc_code, grpc_method, kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{grpc_code}} {{kubernetes_namespace}}/{{grpc_method}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "RPS [$interval]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(grpc_server_handling_seconds_bucket{kubernetes_namespace=~\"$namespace\",$labelselector=\"$labelvalue\"}[$interval])) by (grpc_method, le, kubernetes_namespace))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.99 {{kubernetes_namespace}}/{{grpc_method}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Query Response Time Quantile [$interval]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 33, - "panels": [], - "title": "Alert Sender", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 27, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(thanos_alert_sender_alerts_sent_total{kubernetes_namespace=~\"$namespace\",$labelselector=\"$labelvalue\"}[$interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{alertmanager}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Alerts Sent Rate [$interval]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 37, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(thanos_alert_sender_alerts_dropped_total{kubernetes_namespace=~\"$namespace\",$labelselector=\"$labelvalue\"}[$interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{alertmanager}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Alerts Dropped Rate [$interval]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 35, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(thanos_alert_sender_latency_seconds_bucket{kubernetes_namespace=~\"$namespace\",$labelselector=\"$labelvalue\"}[$interval])) by (alertmanager, le, kubernetes_namespace))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.99 {{kubernetes_namespace}}/{{alertmanager}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Alert Sender Latency Quantile [$interval]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 41, - "panels": [], - "title": "Compaction", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 39, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(prometheus_tsdb_compactions_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_namespace}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Compaction Rate [$interval]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 43, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(prometheus_tsdb_compactions_failed_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_namespace}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Compaction Failure Rate [$interval]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 45, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.99 {{kubernetes_namespace}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Compaction Duration Quantile [$interval]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 25, - "panels": [], - "repeat": null, - "title": "Ops", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_memstats_heap_alloc_bytes{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory Used", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 46 - }, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_gc_duration_seconds{$labelselector=\"$labelvalue\", quantile=\"1\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}} ", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC Time Quantiles", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "refresh": "30s", - "revision": null, - "schemaVersion": 16, - "style": "dark", - "tags": [ - "thanos" - ], - "templating": { - "list": [ - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "1m", - "value": "1m" - }, - "hide": 0, - "label": null, - "name": "interval", - "options": [ - { - "selected": true, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "type": "interval" - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": true, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": "label_values(kubernetes_namespace)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - }, - "hide": 2, - "label": null, - "name": "labelselector", - "options": [ - { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - } - ], - "query": "${VAR_LABELSELECTOR}", - "type": "constant" - }, - { - "current": { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - }, - "hide": 2, - "label": null, - "name": "labelvalue", - "options": [ - { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - } - ], - "query": "${VAR_LABELVALUE}", - "type": "constant" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Thanos Rule", - "uid": "rjUCNfHmz", - "version": 3 -} \ No newline at end of file diff --git a/examples/grafana/thanos-sidecar.json b/examples/grafana/thanos-sidecar.json deleted file mode 100644 index fb91de63b7..0000000000 --- a/examples/grafana/thanos-sidecar.json +++ /dev/null @@ -1,1134 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - }, - { - "name": "VAR_LABELVALUE", - "type": "constant", - "label": "labelvalue", - "value": "prometheus", - "description": "" - }, - { - "name": "VAR_LABELSELECTOR", - "type": "constant", - "label": "labelselector", - "value": "name", - "description": "" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1529906347607, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 22, - "panels": [], - "repeat": null, - "title": "Thanos Sidecar", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "format": "dateTimeFromNow", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 29, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "minSpan": 2, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": null, - "repeatDirection": "h", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "min(thanos_objstore_bucket_last_successful_upload_time{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"})*1000", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Last Upload time", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "format": "dateTimeFromNow", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 30, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "minSpan": 2, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeatDirection": "h", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "min(thanos_sidecar_last_heartbeat_success_time_seconds{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"})*1000", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Last Heartbeat time", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(grpc_server_handled_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method, kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{grpc_code}} {{grpc_method}} {{kubernetes_namespace}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Query RPS [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 32, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(grpc_server_handling_seconds_bucket{$labelselector=\"$labelvalue\"}[$interval])) by (grpc_method, le, kubernetes_namespace))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.99 {{grpc_method}} {{kubernetes_namespace}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Query Response Time Quantile [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(thanos_objstore_bucket_operations_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace, operation)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{operation}} {{kubernetes_namespace}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bucket Operations/s [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 27, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "avg", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (operation,kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{operation}} {{kubernetes_namespace}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bucket Operation Failures/s [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,le))", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "99.99 bucket ops {{kubernetes_namespace}}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bucket Operation Time Quantile [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 25, - "panels": [], - "repeat": null, - "title": "Ops", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_memstats_heap_alloc_bytes{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory Used", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_gc_duration_seconds{$labelselector=\"$labelvalue\", quantile=\"1\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{quantile}} {{kubernetes_namespace}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - }, - { - "expr": "", - "format": "time_series", - "intervalFactor": 1, - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC Time Quantiles", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "refresh": "30s", - "revision": null, - "schemaVersion": 16, - "style": "dark", - "tags": [ - "thanos" - ], - "templating": { - "list": [ - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "1m", - "value": "1m" - }, - "hide": 0, - "label": null, - "name": "interval", - "options": [ - { - "selected": true, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "type": "interval" - }, - { - "current": { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - }, - "hide": 2, - "label": "", - "name": "labelvalue", - "options": [ - { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - } - ], - "query": "${VAR_LABELVALUE}", - "type": "constant" - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": true, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": "label_values(kubernetes_namespace)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - }, - "hide": 2, - "label": null, - "name": "labelselector", - "options": [ - { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - } - ], - "query": "${VAR_LABELSELECTOR}", - "type": "constant" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Thanos Sidecar", - "uid": "IOteEKHik", - "version": 2 -} \ No newline at end of file diff --git a/examples/grafana/thanos-store.json b/examples/grafana/thanos-store.json deleted file mode 100644 index 4027c2da4d..0000000000 --- a/examples/grafana/thanos-store.json +++ /dev/null @@ -1,1299 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - }, - { - "name": "VAR_LABELSELECTOR", - "type": "constant", - "label": "labelselector", - "value": "app", - "description": "" - }, - { - "name": "VAR_LABELVALUE", - "type": "constant", - "label": "labelvalue", - "value": "thanos-store", - "description": "" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1529906350696, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 22, - "panels": [], - "repeat": null, - "title": "thanos-store", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 28, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(grpc_server_handled_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method, kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{grpc_code}} {{grpc_method}} {{kubernetes_namespace}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Query RPS [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 29, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(grpc_server_handling_seconds_bucket{$labelselector=\"$labelvalue\"}[$interval])) by (grpc_method, le, kubernetes_namespace))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{grpc_method}} {{kubernetes_namespace}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Response Time 99.99 Quantile [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 32, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(thanos_bucket_store_sent_chunk_size_bytes_bucket{$labelselector=\"$labelvalue\"}[$interval])) by (le, kubernetes_namespace))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{kubernetes_namespace}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Response Size 99.99 Quantile [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 22 - }, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(thanos_objstore_bucket_operations_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace, operation)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "bucket {{operation}} {{kubernetes_namespace}}", - "refId": "A" - }, - { - "expr": "sum(rate(thanos_bucket_store_block_drops_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "block drops {{kubernetes_namespace}}", - "refId": "B" - }, - { - "expr": "sum(rate(thanos_bucket_store_block_loads_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "block loads {{kubernetes_namespace}}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Operations/s [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 22 - }, - "id": 27, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "avg", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (operation,kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "bucket {{operation}} {{kubernetes_namespace}}", - "refId": "A" - }, - { - "expr": "sum(rate(thanos_bucket_store_block_drop_failures_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "block drop {{kubernetes_namespace}}", - "refId": "B" - }, - { - "expr": "sum(rate(thanos_bucket_store_block_load_failures_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "block load {{kubernetes_namespace}}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Operation Failures/s [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,le, operation))", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "99.99 bucket {{operation}} {{kubernetes_namespace}}", - "refId": "C" - }, - { - "expr": "histogram_quantile(0.9999, sum(rate(thanos_bucket_store_series_get_all_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.99 get all {{kubernetes_namespace}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.9999, sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.99 merge {{kubernetes_namespace}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Operation Time Quantile [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 30, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(thanos_store_index_cache_items_added_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,item_type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "added {{item_type}} {{kubernetes_namespace}}", - "refId": "D" - }, - { - "expr": "sum(rate(thanos_store_index_cache_items_evicted_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,item_type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "evicted {{item_type}} {{kubernetes_namespace}}", - "refId": "E" - }, - { - "expr": "sum(rate(thanos_store_index_cache_requests_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace,item_type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "requests {{item_type}} {{kubernetes_namespace}}", - "refId": "F" - }, - { - "expr": "sum(rate(thanos_store_index_cache_hits_total{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}[$interval])) by (kubernetes_namespace, item_type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "hits {{item_type}} {{kubernetes_namespace}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Cache Ops/s [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "thanos_bucket_store_series_blocks_queried{$labelselector=\"$labelvalue\",quantile=\"0.99\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "blocks queried {{kubernetes_pod_name}} {{kubernetes_namespace}}", - "refId": "C" - }, - { - "expr": "thanos_bucket_store_series_data_fetched{$labelselector=\"$labelvalue\",quantile=\"0.99\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "data fetched {{kubernetes_pod_name}} {{kubernetes_namespace}}", - "refId": "A" - }, - { - "expr": "thanos_bucket_store_series_result_series{$labelselector=\"$labelvalue\",quantile=\"0.99\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "result series {{kubernetes_pod_name}} {{kubernetes_namespace}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Pod Operation Time 99th Quantile [$interval]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 25, - "panels": [], - "repeat": null, - "title": "Ops", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_memstats_heap_alloc_bytes{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory Used", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{$labelselector=\"$labelvalue\",kubernetes_namespace=~\"$namespace\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 51 - }, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_gc_duration_seconds{$labelselector=\"$labelvalue\", quantile=\"1\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kubernetes_namespace}} {{kubernetes_pod_name}} ", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC Time Quantiles", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "refresh": "30s", - "revision": null, - "schemaVersion": 16, - "style": "dark", - "tags": [ - "thanos" - ], - "templating": { - "list": [ - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "1m", - "value": "1m" - }, - "hide": 0, - "label": null, - "name": "interval", - "options": [ - { - "selected": true, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "type": "interval" - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": true, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": "label_values(kubernetes_namespace)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - }, - "hide": 2, - "label": null, - "name": "labelselector", - "options": [ - { - "value": "${VAR_LABELSELECTOR}", - "text": "${VAR_LABELSELECTOR}" - } - ], - "query": "${VAR_LABELSELECTOR}", - "type": "constant" - }, - { - "current": { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - }, - "hide": 2, - "label": null, - "name": "labelvalue", - "options": [ - { - "value": "${VAR_LABELVALUE}", - "text": "${VAR_LABELVALUE}" - } - ], - "query": "${VAR_LABELVALUE}", - "type": "constant" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Thanos Store", - "uid": "KqPVKRIiz", - "version": 2 -} \ No newline at end of file diff --git a/jsonnetfile.json b/jsonnetfile.json new file mode 100644 index 0000000000..1bcb73e8a9 --- /dev/null +++ b/jsonnetfile.json @@ -0,0 +1,33 @@ +{ + "dependencies": [ + { + "name": "thanos-mixin", + "source": { + "local": { + "directory": "mixin/thanos" + } + }, + "version": "." + }, + { + "name": "grafonnet", + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet-lib", + "subdir": "grafonnet" + } + }, + "version": "master" + }, + { + "name": "grafana-builder", + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs", + "subdir": "grafana-builder" + } + }, + "version": "master" + } + ] +} diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json new file mode 100644 index 0000000000..ae23cf5774 --- /dev/null +++ b/jsonnetfile.lock.json @@ -0,0 +1,35 @@ +{ + "dependencies": [ + { + "name": "grafana-builder", + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs", + "subdir": "grafana-builder" + } + }, + "version": "f4c59f64f80442f871a06c91edf74d014b82acaf", + "sum": "ELsYwK+kGdzX1mee2Yy+/b2mdO4Y503BOCDkFzwmGbE=" + }, + { + "name": "grafonnet", + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet-lib", + "subdir": "grafonnet" + } + }, + "version": "69bc267211790a1c3f4ea6e6211f3e8ffe22f987", + "sum": "BjHfWzqSAgtAKEVD6ipoYOkb8XT5wSBIboY4ZLwhlOU=" + }, + { + "name": "thanos-mixin", + "source": { + "local": { + "directory": "mixin/thanos" + } + }, + "version": "" + } + ] +} diff --git a/mixin/thanos-grafana-builder/builder.libsonnet b/mixin/thanos-grafana-builder/builder.libsonnet new file mode 100644 index 0000000000..a68e8065b3 --- /dev/null +++ b/mixin/thanos-grafana-builder/builder.libsonnet @@ -0,0 +1,191 @@ +local grafana = import 'grafonnet/grafana.libsonnet'; +local template = grafana.template; + +(import 'grafana-builder/grafana.libsonnet') + +{ + collapse: { + collapse: true, + }, + + panel(title, description=null):: + super.panel(title) { [if description != null then 'description']: description }, + + addDashboardLink(name): { + links+: [ + { + dashboard: name, + includeVars: true, + keepTime: true, + title: name, + type: 'dashboard', + }, + ], + }, + + template(name, metricName, selector='', includeAll=false, allValues=''):: + local t = if includeAll then + template.new( + name, + '$datasource', + 'label_values(%s{%s}, %s)' % [metricName, selector, name], + label=name, + refresh=1, + sort=2, + current='all', + allValues=allValues, + includeAll=true + ) + else + template.new( + name, + '$datasource', + 'label_values(%s{%s}, %s)' % [metricName, selector, name], + label=name, + refresh=1, + sort=2, + ); + + { + templating+: { + list+: [ + t, + ], + }, + }, + + spanSize(size):: { + span: size, + }, + + postfix(postfix):: { + postfix: postfix, + }, + + sparkline:: { + sparkline: { + show: true, + lineColor: 'rgb(31, 120, 193)', + fillColor: 'rgba(31, 118, 189, 0.18)', + }, + }, + + latencyPanel(metricName, selector, multiplier='1'):: { + nullPointMode: 'null as zero', + targets: [ + { + expr: 'histogram_quantile(0.99, sum(rate(%s_bucket{%s}[$interval])) by (job, le)) * %s' % [metricName, selector, multiplier], + format: 'time_series', + intervalFactor: 2, + legendFormat: 'P99 {{job}}', + refId: 'A', + step: 10, + }, + { + expr: 'sum(rate(%s_sum{%s}[$interval])) by (job) * %s / sum(rate(%s_count{%s}[$interval])) by (job)' % [metricName, selector, multiplier, metricName, selector], + format: 'time_series', + intervalFactor: 2, + legendFormat: 'mean {{job}}', + refId: 'B', + step: 10, + }, + { + expr: 'histogram_quantile(0.50, sum(rate(%s_bucket{%s}[$interval])) by (job, le)) * %s' % [metricName, selector, multiplier], + format: 'time_series', + intervalFactor: 2, + legendFormat: 'P50 {{job}}', + refId: 'C', + step: 10, + }, + ], + yaxes: $.yaxes('s'), + }, + + qpsErrTotalPanel(selectorErr, selectorTotal):: { + local expr(selector) = 'sum(rate(' + selector + '[$interval]))', // {{job}} + + aliasColors: { + 'error': '#E24D42', + }, + targets: [ + { + expr: '%s / %s' % [expr(selectorErr), expr(selectorTotal)], + format: 'time_series', + intervalFactor: 2, + legendFormat: 'error', + refId: 'A', + step: 10, + }, + ], + yaxes: $.yaxes({ format: 'percentunit' }), + } + $.stack, + + qpsSuccErrRatePanel(selectorErr, selectorTotal):: { + local expr(selector) = 'sum(rate(' + selector + '[$interval]))', // {{job}} + + aliasColors: { + success: '#7EB26D', + 'error': '#E24D42', + }, + targets: [ + { + expr: '%s / %s' % [expr(selectorErr), expr(selectorTotal)], + format: 'time_series', + intervalFactor: 2, + legendFormat: 'error', + refId: 'A', + step: 10, + }, + { + expr: '(%s - %s) / %s' % [expr(selectorTotal), expr(selectorErr), expr(selectorTotal)], + format: 'time_series', + intervalFactor: 2, + legendFormat: 'success', + refId: 'B', + step: 10, + }, + ], + yaxes: $.yaxes({ format: 'percentunit', max: 1 }), + } + $.stack, + + resourceUtilizationRow():: + $.row('Resources') + .addPanel( + $.panel('Memory Used') + + $.queryPanel( + [ + 'go_memstats_alloc_bytes{namespace="$namespace",job=~"$job",kubernetes_pod_name=~"$pod"}', + 'go_memstats_heap_alloc_bytes{namespace="$namespace",job=~"$job",kubernetes_pod_name=~"$pod"}', + 'rate(go_memstats_alloc_bytes_total{namespace="$namespace",job=~"$job",kubernetes_pod_name=~"$pod"}[30s])', + 'rate(go_memstats_heap_alloc_bytes{namespace="$namespace",job=~"$job",kubernetes_pod_name=~"$pod"}[30s])', + 'go_memstats_stack_inuse_bytes{namespace="$namespace",job=~"$job",kubernetes_pod_name=~"$pod"}', + 'go_memstats_heap_inuse_bytes{namespace="$namespace",job=~"$job",kubernetes_pod_name=~"$pod"}', + ], + [ + 'alloc all {{pod}}', + 'alloc heap {{pod}}', + 'alloc rate all {{pod}}', + 'alloc rate heap {{pod}}', + 'inuse stack {{pod}}', + 'inuse heap {{pod}}', + ] + ), + ) + .addPanel( + $.panel('Goroutines') + + $.queryPanel( + 'go_goroutines{namespace="$namespace",job=~"$job"}', + '{{pod}}' + ) + ) + .addPanel( + $.panel('GC Time Quantiles') + + $.queryPanel( + 'go_gc_duration_seconds{namespace="$namespace",job=~"$job",kubernetes_pod_name=~"$pod"}', + '{{quantile}} {{pod}}' + ) + ) + + $.collapse, +} + +(import 'grpc.libsonnet') + +(import 'http.libsonnet') + +(import 'slo.libsonnet') diff --git a/mixin/thanos-grafana-builder/grpc.libsonnet b/mixin/thanos-grafana-builder/grpc.libsonnet new file mode 100644 index 0000000000..a3a981fd18 --- /dev/null +++ b/mixin/thanos-grafana-builder/grpc.libsonnet @@ -0,0 +1,107 @@ +{ + grpcQpsPanel(type, selector):: { + local prefix = if type == 'client' then 'grpc_client' else 'grpc_server', + + aliasColors: { + Aborted: '#EAB839', + AlreadyExists: '#7EB26D', + FailedPrecondition: '#6ED0E0', + Unimplemented: '#6ED0E0', + InvalidArgument: '#EF843C', + NotFound: '#EF843C', + PermissionDenied: '#EF843C', + Unauthenticated: '#EF843C', + Canceled: '#E24D42', + DataLoss: '#E24D42', + DeadlineExceeded: '#E24D42', + Internal: '#E24D42', + OutOfRange: '#E24D42', + ResourceExhausted: '#E24D42', + Unavailable: '#E24D42', + Unknown: '#E24D42', + OK: '#7EB26D', + 'error': '#E24D42', + }, + targets: [ + { + expr: 'sum(rate(%s_handled_total{%s}[$interval])) by (job, grpc_code)' % [prefix, selector], + format: 'time_series', + intervalFactor: 2, + legendFormat: '{{job}} {{grpc_code}}', + refId: 'A', + step: 10, + }, + ], + } + $.stack, + + grpcQpsPanelDetailed(type, selector):: { + local prefix = if type == 'client' then 'grpc_client' else 'grpc_server', + targets: [ + { + expr: 'sum(rate(%s_handled_total{%s}[$interval])) by (job, grpc_method, grpc_code)' % [prefix, selector], + format: 'time_series', + intervalFactor: 2, + legendFormat: '{{job}} {{grpc_method}} {{grpc_code}}', + refId: 'A', + step: 10, + }, + ], + } + $.stack, + + grpcErrorsPanel(type, selector):: + local prefix = if type == 'client' then 'grpc_client' else 'grpc_server'; + $.qpsErrTotalPanel( + '%s_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable",%s}' % [prefix, selector], + '%s_started_total{%s}' % [prefix, selector], + ), + + grpcErrDetailsPanel(type, selector):: + local prefix = if type == 'client' then 'grpc_client' else 'grpc_server'; + $.queryPanel( + ||| + sum(rate(%s_handled_total{grpc_code!="OK",%s}[$interval])) by (job, grpc_method, grpc_code) + ||| % [prefix, selector], + '{{job}} {{grpc_method}} {{grpc_code}}' + ) + + $.stack, + + grpcLatencyPanel(type, selector, multiplier='1'):: + local prefix = if type == 'client' then 'grpc_client' else 'grpc_server'; + $.queryPanel( + [ + 'histogram_quantile(0.99, sum(rate(%s_handling_seconds_bucket{%s}[$interval])) by (job, le)) * %s' % [prefix, selector, multiplier], + ||| + sum(rate(%s_handling_seconds_sum{%s}[$interval])) by (job) * %s + / + sum(rate(%s_handling_seconds_count{%s}[$interval])) by (job) + ||| % [prefix, selector, multiplier, prefix, selector], + 'histogram_quantile(0.50, sum(rate(%s_handling_seconds_bucket{%s}[$interval])) by (job, le)) * %s' % [prefix, selector, multiplier], + ], + [ + 'P99 {{job}}', + 'mean {{job}}', + 'P50 {{job}}', + ] + ) + + { yaxes: $.yaxes('s') }, + + grpcLatencyPanelDetailed(type, selector, multiplier='1'):: + local prefix = if type == 'client' then 'grpc_client' else 'grpc_server'; + $.queryPanel( + [ + 'histogram_quantile(0.99, sum(rate(%s_handling_seconds_bucket{%s}[$interval])) by (job, grpc_method, le)) * %s' % [prefix, selector, multiplier], + ||| + sum(rate(%s_handling_seconds_sum{%s}[$interval])) by (job) * %s + / + sum(rate(%s_handling_seconds_count{%s}[$interval])) by (job) + ||| % [prefix, selector, multiplier, prefix, selector], + 'histogram_quantile(0.50, sum(rate(%s_handling_seconds_bucket{%s}[$interval])) by (job, grpc_method, le)) * %s' % [prefix, selector, multiplier], + ], + [ + 'P99 {{job}} {{grpc_method}}', + 'mean {{job}} {{grpc_method}}', + 'P50 {{job}} {{grpc_method}}', + ] + ) + + { yaxes: $.yaxes('s') }, +} diff --git a/mixin/thanos-grafana-builder/http.libsonnet b/mixin/thanos-grafana-builder/http.libsonnet new file mode 100644 index 0000000000..a03682b3dd --- /dev/null +++ b/mixin/thanos-grafana-builder/http.libsonnet @@ -0,0 +1,82 @@ +{ + httpQpsPanel(metricName, selector):: { + aliasColors: { + '1xx': '#EAB839', + '2xx': '#7EB26D', + '3xx': '#6ED0E0', + '4xx': '#EF843C', + '5xx': '#E24D42', + success: '#7EB26D', + 'error': '#E24D42', + }, + targets: [ + { + expr: 'sum(label_replace(rate(%s{%s}[$interval]),"status_code", "${1}xx", "code", "([0-9])..")) by (job, status_code)' % [metricName, selector], + format: 'time_series', + intervalFactor: 2, + legendFormat: '{{job}} {{status_code}}', + refId: 'A', + step: 10, + }, + ], + } + $.stack, + + httpQpsPanelDetailed(metricName, selector):: + $.httpQpsPanel(metricName, selector) { + targets: [ + { + expr: 'sum(label_replace(rate(%s{%s}[$interval]),"status_code", "${1}xx", "code", "([0-9])..")) by (job, handler, status_code)' % [metricName, selector], + format: 'time_series', + intervalFactor: 2, + legendFormat: '{{job}} {{handler}} {{status_code}}', + refId: 'A', + step: 10, + }, + ], + }, + + httpErrPanel(metricName, selector):: + $.qpsErrTotalPanel( + '%s{%s,code=~"5.."}' % [metricName, selector], + '%s{%s}' % [metricName, selector], + ), + + httpErrDetailsPanel(metricName, selector):: + $.queryPanel( + 'sum(rate(%s{%s,code!~"2.."}[$interval])) by (job, handler, code)' % [metricName, selector], + '{{job}} {{handler}} {{code}}' + ) + + { yaxes: $.yaxes({ format: 'percentunit' }) } + + $.stack, + + httpLatencyDetailsPanel(metricName, selector, multiplier='1'):: { + nullPointMode: 'null as zero', + targets: [ + { + expr: 'histogram_quantile(0.99, sum(rate(%s_bucket{%s}[$interval])) by (job, handler, le)) * %s' % [metricName, selector, multiplier], + format: 'time_series', + intervalFactor: 2, + legendFormat: 'P99 {{job}} {{handler}}', + refId: 'A', + step: 10, + }, + { + expr: 'sum(rate(%s_sum{%s}[$interval])) by (job, handler) * %s / sum(rate(%s_count{%s}[$interval])) by (job, handler)' % [metricName, selector, multiplier, metricName, selector], + format: 'time_series', + intervalFactor: 2, + legendFormat: 'mean {{job}} {{handler}}', + refId: 'B', + step: 10, + }, + { + expr: 'histogram_quantile(0.50, sum(rate(%s_bucket{%s}[$interval])) by (job, handler, le)) * %s' % [metricName, selector, multiplier], + format: 'time_series', + intervalFactor: 2, + legendFormat: 'P50 {{job}} {{handler}}', + refId: 'C', + step: 10, + }, + ], + yaxes: $.yaxes('s'), + }, +} diff --git a/mixin/thanos-grafana-builder/slo.libsonnet b/mixin/thanos-grafana-builder/slo.libsonnet new file mode 100644 index 0000000000..bee885037a --- /dev/null +++ b/mixin/thanos-grafana-builder/slo.libsonnet @@ -0,0 +1,29 @@ +{ + sloLatency(title, description, selector, quantile, warning, critical):: + $.panel(title, description) + + $.queryPanel( + 'histogram_quantile(%.2f, sum(rate(%s[$interval])) by (job, le))' % [quantile, selector], + '{{job}} P' + quantile * 100 + ) + + { + yaxes: $.yaxes('s'), + thresholds+: [ + { + value: warning, + colorMode: 'warning', + op: 'gt', + fill: true, + line: true, + yaxis: 'left', + }, + { + value: critical, + colorMode: 'critical', + op: 'gt', + fill: true, + line: true, + yaxis: 'left', + }, + ], + }, +} diff --git a/mixin/thanos/README.md b/mixin/thanos/README.md new file mode 100644 index 0000000000..1472b88db8 --- /dev/null +++ b/mixin/thanos/README.md @@ -0,0 +1,158 @@ +# thanos-mixin + +> Note that everything is experimental and may change significantly at any time. +> Also it still has missing alert and dashboard definitions for certain components, e.g. rule and sidecar. Please feel free to contribute. + +This directory contains extensible and customizable monitoring definitons for Thanos. [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy monitoring experience for Thanos. + +You can find more about monitoring-mixins in [the design document](https://docs.google.com/document/d/1A9xvzwqnFVSOZ5fD3blKODXfsat5fg6ZhnKu9LK3lB4/edit#heading=h.gt9r2h2gklj3), and you could check out other examples like [Prometheus Mixin](https://github.com/prometheus/prometheus/tree/master/documentation/prometheus-mixin). + +The content of this project is written in [jsonnet](http://jsonnet.org/). This project could both be described as a package as well as a library. + +## Requirements + +### jsonnet + +The content of this project consists of a set of [jsonnet](http://jsonnet.org/) files making up a library to be consumed. + +We recommend to use [go-jsonnet](https://github.com/google/go-jsonnet). It's an implementation of [Jsonnet](http://jsonnet.org/) in pure Go. It is feature complete but is not as heavily exercised as the [Jsonnet C++ implementation](https://github.com/google/jsonnet). + +To install: + +```shell +go get github.com/google/go-jsonnet/cmd/jsonnet +``` + +### jsonnet-bundler + +`thanos-mixin` uses [jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler#install) (the jsonnet package manager) to manage its dependencies. + +We also recommend you to use `jsonnet-bundler` to install or update if you decide to use `thanos-mixin` as a dependency for your custom configurations. + +To install: + +```shell +go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb +``` + +> An e.g. of how to install a given version of this library: `jb install github.com/thanos-io/thanos/jsonnet/thanos-mixin@master`. + +## Use as a library + +To use the `thanos-mixin` as a dependency, simply use the `jsonnet-bundler` install functionality: +```shell +$ mkdir thanos-mixin; cd thanos-mixin +$ jb init # Creates the initial/empty `jsonnetfile.json` +# Install the thanos-mixin dependency +$ jb install github.com/thanos-io/thanos/jsonnet/thanos-mixin@master # Creates `vendor/` & `jsonnetfile.lock.json`, and fills in `jsonnetfile.json` +``` + +To update the `thanos-mixin` as a dependency, simply use the `jsonnet-bundler` update functionality: +```shell +$ jb update +``` + +#### Configure + +This project is intended to be used as a library. You can extend and customize dashboards and alerting rules by creating for own generators, such as the generators ([alerts.jsonnet](alerts.jsonnet) and [dashboards.jsonnet](dashboards.jsonnet)) that are use to create [examples](examples). Default parameters are collected in [defaults.jsonnet](defaults.jsonnet), feel free to modify and generate your own definitons. + +[embedmd]:# (defaults.libsonnet) +```libsonnet +{ + querier+:: { + jobPrefix: 'thanos-querier', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sQuerier' % $.dashboard.prefix, + }, + store+:: { + jobPrefix: 'thanos-store', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sStore' % $.dashboard.prefix, + }, + receiver+:: { + jobPrefix: 'thanos-receiver', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sReceiver' % $.dashboard.prefix, + }, + ruler+:: { + jobPrefix: 'thanos-ruler', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sRuler' % $.dashboard.prefix, + }, + compactor+:: { + jobPrefix: 'thanos-compactor', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sCompactor' % $.dashboard.prefix, + }, + sidecar+:: { + jobPrefix: 'thanos-sidecar', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sSidecar' % $.dashboard.prefix, + }, + overview+:: { + title: '%(prefix)sOverview' % $.dashboard.prefix, + }, + dashboard+:: { + prefix: 'Thanos / ', + tags: ['thanos-mixin'], + namespaceQuery: 'kube_pod_info', + }, +} +``` + +You can format your code using: +```shell +$ make jsonnet-format +``` + +## Examples + +This project is intended to be used as a library. However, it also provides drop-in examples to monitor Thanos. + +### Requirements + +Although all the required dependencies are handled by `Makefile`, keep in mind that in addition the dependencies that are listed above we have following dependencies: + +#### gojsontoyaml + +`gojsontoyaml` is used to convert generated `json` definitons to `yaml`. + +To install: +```shell +go get github.com/brancz/gojsontoyaml +``` + +### Generate + +To generate examples after modifying, make sure `jsonnet` dependencies are installed. +```shell +$ make jsonnet-vendor +``` + +and then + +```shell +$ make examples +``` + +Make action runs the jsonnet code, then reads each key of the generated json and uses that as the file name, and writes the value of that key to that file, and converts each json manifest to yaml. + +> Make commands should handle dependecies for you. + +### Test and validate + +You validate your structural correctness of your Prometheus [alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) or [recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) with: + +```shell +$ make example-rules-lint +``` + +Check out [test.yaml](examples/alerts/tests.yaml) to add/modify tests for the mixin. To learn more about how to write test for Prometheus, check out [official documentation](https://www.prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/). + +You test alerts with: + +```shell +$ make alerts-test +``` + +--- diff --git a/mixin/thanos/alerts.jsonnet b/mixin/thanos/alerts.jsonnet new file mode 100644 index 0000000000..574da7f5a6 --- /dev/null +++ b/mixin/thanos/alerts.jsonnet @@ -0,0 +1,4 @@ +( + (import 'mixin.libsonnet') + + (import 'defaults.libsonnet') +).prometheusAlerts diff --git a/mixin/thanos/alerts/absent.libsonnet b/mixin/thanos/alerts/absent.libsonnet new file mode 100644 index 0000000000..a134fc8f60 --- /dev/null +++ b/mixin/thanos/alerts/absent.libsonnet @@ -0,0 +1,37 @@ +{ + local thanos = self, + + // We build alerts for the presence of all these jobs. + jobs:: { + ThanosQuerier: thanos.querier.selector, + ThanosStore: thanos.store.selector, + ThanosReceiver: thanos.receiver.selector, + ThanosRuler: thanos.ruler.selector, + ThanosCompactor: thanos.compactor.selector, + ThanosSidecar: thanos.sidecar.selector, + }, + + prometheusAlerts+:: { + groups+: [ + { + name: 'thanos-component-absent.rules', + rules: [ + { + alert: '%sIsDown' % name, + expr: ||| + absent(up{%s} == 1) + ||| % thanos.jobs[name], + 'for': '5m', + labels: { + severity: 'critical', + }, + annotations: { + message: '%s has disappeared from Prometheus target discovery.' % name, + }, + } + for name in std.objectFields(thanos.jobs) + ], + }, + ], + }, +} diff --git a/mixin/thanos/alerts/alerts.libsonnet b/mixin/thanos/alerts/alerts.libsonnet new file mode 100644 index 0000000000..0eb63dc98d --- /dev/null +++ b/mixin/thanos/alerts/alerts.libsonnet @@ -0,0 +1,6 @@ +(import 'compactor.libsonnet') + +(import 'querier.libsonnet') + +(import 'receiver.libsonnet') + +(import 'sidecar.libsonnet') + +(import 'store.libsonnet') + +(import 'absent.libsonnet') diff --git a/mixin/thanos/alerts/compactor.libsonnet b/mixin/thanos/alerts/compactor.libsonnet new file mode 100644 index 0000000000..6c34259b9b --- /dev/null +++ b/mixin/thanos/alerts/compactor.libsonnet @@ -0,0 +1,84 @@ +{ + local thanos = self, + compactor+:: { + jobPrefix: error 'must provide job prefix for Thanos Compact alerts', + selector: error 'must provide selector for Thanos Compact alerts', + }, + prometheusAlerts+:: { + groups+: [ + { + name: 'thanos-compactor.rules', + rules: [ + { + alert: 'ThanosCompactorMultipleCompactsAreRunning', + annotations: { + message: 'You should never run more than one Thanos Compact at once. You have {{ $value }}', + }, + expr: 'sum(up{%(selector)s}) > 1' % thanos.compactor, + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosCompactorHalted', + annotations: { + message: 'Thanos Compact {{$labels.job}} has failed to run and now is halted.', + }, + expr: 'thanos_compactor_halted{%(selector)s} == 1' % thanos.compactor, + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosCompactorHighCompactionFailures', + annotations: { + message: 'Thanos Compact {{$labels.job}} is failing to execute {{ $value | humanize }}% of compactions.', + }, + expr: ||| + ( + sum by (job) (rate(thanos_compact_group_compactions_failures_total{%(selector)s}[5m])) + / + sum by (job) (rate(thanos_compact_group_compactions_total{%(selector)s}[5m])) + * 100 > 5 + ) + ||| % thanos.compactor, + 'for': '15m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosCompactorBucketHighOperationFailures', + annotations: { + message: 'Thanos Compact {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations.', + }, + expr: ||| + ( + sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{%(selector)s}[5m])) + / + sum by (job) (rate(thanos_objstore_bucket_operations_total{%(selector)s}[5m])) + * 100 > 5 + ) + ||| % thanos.compactor, + 'for': '15m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosCompactorHasNotRun', + annotations: { + message: 'Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours.', + }, + expr: '(time() - max(thanos_objstore_bucket_last_successful_upload_time{%(selector)s})) / 60 / 60 > 24' % thanos.compactor, + labels: { + severity: 'warning', + }, + }, + ], + }, + ], + }, +} diff --git a/mixin/thanos/alerts/querier.libsonnet b/mixin/thanos/alerts/querier.libsonnet new file mode 100644 index 0000000000..38705aacf3 --- /dev/null +++ b/mixin/thanos/alerts/querier.libsonnet @@ -0,0 +1,138 @@ +{ + local thanos = self, + querier+:: { + jobPrefix: error 'must provide job prefix for Thanos Query alerts', + selector: error 'must provide selector for Thanos Query alerts', + }, + prometheusAlerts+:: { + groups+: [ + { + name: 'thanos-querier.rules', + rules: [ + { + alert: 'ThanosQuerierHttpRequestQueryErrorRateHigh', + annotations: { + message: 'Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests.', + }, + expr: ||| + ( + sum(rate(http_requests_total{code=~"5..", %(selector)s, handler="query"}[5m])) + / + sum(rate(http_requests_total{%(selector)s, handler="query"}[5m])) + ) * 100 > 5 + ||| % thanos.querier, + 'for': '5m', + labels: { + severity: 'critical', + }, + }, + { + alert: 'ThanosQuerierHttpRequestQueryRangeErrorRateHigh', + annotations: { + message: 'Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests.', + }, + expr: ||| + ( + sum(rate(http_requests_total{code=~"5..", %(selector)s, handler="query_range"}[5m])) + / + sum(rate(http_requests_total{%(selector)s, handler="query_range"}[5m])) + ) * 100 > 5 + ||| % thanos.querier, + 'for': '5m', + labels: { + severity: 'critical', + }, + }, + { + alert: 'ThanosQuerierGrpcServerErrorRate', + annotations: { + message: 'Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', + }, + expr: ||| + ( + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", %(selector)s}[5m])) + / + sum by (job) (rate(grpc_server_started_total{%(selector)s}[5m])) + * 100 > 5 + ) + ||| % thanos.querier, + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosQuerierGrpcClientErrorRate', + annotations: { + message: 'Thanos Query {{$labels.job}} is failing to send {{ $value | humanize }}% of requests.', + }, + expr: ||| + ( + sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", %(selector)s}[5m])) + / + sum by (job) (rate(grpc_client_started_total{%(selector)s}[5m])) + * 100 > 5 + ) + ||| % thanos.querier, + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosQuerierHighDNSFailures', + annotations: { + message: 'Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries.', + }, + expr: ||| + ( + sum by (job) (rate(thanos_query_store_apis_dns_failures_total{%(selector)s}[5m])) + / + sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{%(selector)s}[5m])) + > 1 + ) + ||| % thanos.querier, + 'for': '15m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosQuerierInstantLatencyHigh', + annotations: { + message: 'Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', + }, + expr: ||| + ( + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{%(selector)s, handler="query"})) > 10 + and + sum by (job) (rate(http_request_duration_seconds_bucket{%(selector)s, handler="query"}[5m])) > 0 + ) + ||| % thanos.querier, + 'for': '10m', + labels: { + severity: 'critical', + }, + }, + { + alert: 'ThanosQuerierRangeLatencyHigh', + annotations: { + message: 'Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', + }, + expr: ||| + ( + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{%(selector)s, handler="query_range"})) > 10 + and + sum by (job) (rate(http_request_duration_seconds_count{%(selector)s, handler="query_range"}[5m])) > 0 + ) + ||| % thanos.querier, + 'for': '10m', + labels: { + severity: 'critical', + }, + }, + ], + }, + ], + }, +} diff --git a/mixin/thanos/alerts/receiver.libsonnet b/mixin/thanos/alerts/receiver.libsonnet new file mode 100644 index 0000000000..9441bcaf41 --- /dev/null +++ b/mixin/thanos/alerts/receiver.libsonnet @@ -0,0 +1,97 @@ +{ + local thanos = self, + receiver+:: { + jobPrefix: error 'must provide job prefix for Thanos Receive alerts', + selector: error 'must provide selector for Thanos Receive alerts', + }, + prometheusAlerts+:: { + groups+: [ + { + name: 'thanos-receiver.rules', + rules: [ + { + alert: 'ThanosReceiverHttpRequestErrorRateHigh', + annotations: { + message: 'Thanos Receive {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', + }, + expr: ||| + ( + sum(rate(http_requests_total{code=~"5..", %(selector)s, handler="receive"}[5m])) + / + sum(rate(http_requests_total{%(selector)s, handler="receive"}[5m])) + ) * 100 > 5 + ||| % thanos.receiver, + 'for': '5m', + labels: { + severity: 'critical', + }, + }, + { + alert: 'ThanosReceiverHttpRequestLatencyHigh', + annotations: { + message: 'Thanos Receive {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for requests.', + }, + expr: ||| + ( + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{%(selector)s, handler="receive"})) > 10 + and + sum by (job) (rate(http_request_duration_seconds_count{%(selector)s, handler="receive"}[5m])) > 0 + ) + ||| % thanos.receiver, + 'for': '10m', + labels: { + severity: 'critical', + }, + }, + { + alert: 'ThanosReceiverHighForwardRequestFailures', + annotations: { + message: 'Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize }}% of requests.', + }, + expr: ||| + ( + sum by (job) (rate(thanos_receive_forward_requests_total{result="error", %(selector)s}[5m])) + / + sum by (job) (rate(thanos_receive_forward_requests_total{%(selector)s}[5m])) + * 100 > 5 + ) + ||| % thanos.receiver, + 'for': '5m', + labels: { + severity: 'critical', + }, + }, + { + alert: 'ThanosReceiverHighHashringFileRefreshFailures', + annotations: { + message: 'Thanos Receive {{$labels.job}} is failing to refresh hashring file, {{ $value | humanize }} of attempts failed.', + }, + expr: ||| + ( + sum by (job) (rate(thanos_receive_hashrings_file_errors_total{%(selector)s}[5m])) + / + sum by (job) (rate(thanos_receive_hashrings_file_refreshes_total{%(selector)s}[5m])) + > 0 + ) + ||| % thanos.receiver, + 'for': '15m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosReceiverConfigReloadFailure', + annotations: { + message: 'Thanos Receive {{$labels.job}} has not been able to reload hashring configurations.', + }, + expr: 'avg(thanos_receive_config_last_reload_successful{%(selector)s}) by (job) != 1' % thanos.receiver, + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + ], + }, + ], + }, +} diff --git a/mixin/thanos/alerts/sidecar.libsonnet b/mixin/thanos/alerts/sidecar.libsonnet new file mode 100644 index 0000000000..7c80e3af08 --- /dev/null +++ b/mixin/thanos/alerts/sidecar.libsonnet @@ -0,0 +1,28 @@ +{ + local thanos = self, + sidecar+:: { + jobPrefix: error 'must provide job prefix for Thanos Sidecar alerts', + selector: error 'must provide selector for Thanos Sidecar alerts', + }, + prometheusAlerts+:: { + groups+: [ + { + name: 'thanos-sidecar.rules', + rules: [ + { + alert: 'ThanosSidecarUnhealthy', + annotations: { + message: 'Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for {{ $value }} seconds.', + }, + expr: ||| + count(time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{%(selector)s}) by (job, pod) >= 300) > 0 + ||| % thanos.sidecar, + labels: { + severity: 'critical', + }, + }, + ], + }, + ], + }, +} diff --git a/mixin/thanos/alerts/store.libsonnet b/mixin/thanos/alerts/store.libsonnet new file mode 100644 index 0000000000..77ff1baedd --- /dev/null +++ b/mixin/thanos/alerts/store.libsonnet @@ -0,0 +1,86 @@ +{ + local thanos = self, + store+:: { + jobPrefix: error 'must provide job prefix for Thanos Store alerts', + selector: error 'must provide selector for Thanos Store alerts', + }, + prometheusAlerts+:: { + groups+: [ + { + name: 'thanos-store.rules', + rules: [ + { + alert: 'ThanosStoreGrpcErrorRate', + annotations: { + message: 'Thanos Store {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', + }, + expr: ||| + ( + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", %(selector)s}[5m])) + / + sum by (job) (rate(grpc_server_started_total{%(selector)s}[5m])) + * 100 > 5 + ) + ||| % thanos.store, + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosStoreSeriesGateLatencyHigh', + annotations: { + message: 'Thanos Store {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for store series gate requests.', + }, + expr: ||| + ( + histogram_quantile(0.9, sum by (job, le) (thanos_bucket_store_series_gate_duration_seconds_bucket{%(selector)s})) > 2 + and + sum by (job) (rate(thanos_bucket_store_series_gate_duration_seconds_count{%(selector)s}[5m])) > 0 + ) + ||| % thanos.store, + 'for': '10m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosStoreBucketHighOperationFailures', + annotations: { + message: 'Thanos Store {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations.', + }, + expr: ||| + ( + sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{%(selector)s}[5m])) + / + sum by (job) (rate(thanos_objstore_bucket_operations_total{%(selector)s}[5m])) + * 100 > 5 + ) + ||| % thanos.store, + 'for': '15m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosStoreObjstoreOperationLatencyHigh', + annotations: { + message: 'Thanos Store {{$labels.job}} Bucket has a 99th percentile latency of {{ $value }} seconds for the bucket operations.', + }, + expr: ||| + ( + histogram_quantile(0.9, sum by (job, le) (thanos_objstore_bucket_operation_duration_seconds_bucket{%(selector)s})) > 15 + and + sum by (job) (rate(thanos_objstore_bucket_operation_duration_seconds_count{%(selector)s}[5m])) > 0 + ) + ||| % thanos.store, + 'for': '10m', + labels: { + severity: 'warning', + }, + }, + ], + }, + ], + }, +} diff --git a/mixin/thanos/dashboards.jsonnet b/mixin/thanos/dashboards.jsonnet new file mode 100644 index 0000000000..a9cd0bbfcf --- /dev/null +++ b/mixin/thanos/dashboards.jsonnet @@ -0,0 +1,9 @@ +local dashboards = ( + (import 'mixin.libsonnet') + + (import 'defaults.libsonnet') +).grafanaDashboards; + +{ + [name]: dashboards[name] + for name in std.objectFields(dashboards) +} diff --git a/mixin/thanos/dashboards/compactor.libsonnet b/mixin/thanos/dashboards/compactor.libsonnet new file mode 100644 index 0000000000..79f594395a --- /dev/null +++ b/mixin/thanos/dashboards/compactor.libsonnet @@ -0,0 +1,165 @@ +local g = import '../thanos-grafana-builder/builder.libsonnet'; + +{ + local thanos = self, + compactor+:: { + jobPrefix: error 'must provide job prefix for Thanos Compact dashboard', + selector: error 'must provide selector for Thanos Compact dashboard', + title: error 'must provide title for Thanos Compact dashboard', + }, + grafanaDashboards+:: { + 'compactor.json': + g.dashboard(thanos.compactor.title) + .addRow( + g.row('Group Compaction') + .addPanel( + g.panel( + 'Rate', + 'Shows rate of execution for compactions against blocks that are stored in the bucket by compaction group.' + ) + + g.queryPanel( + 'sum(rate(thanos_compact_group_compactions_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, group)', + 'compaction {{job}} {{group}}' + ) + + g.stack + ) + .addPanel( + g.panel( + 'Errors', + 'Shows ratio of errors compared to the total number of executed compactions against blocks that are stored in the bucket.' + ) + + g.qpsErrTotalPanel( + 'thanos_compact_group_compactions_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_compact_group_compactions_total{namespace="$namespace",job=~"$job"}', + ) + ) + ) + .addRow( + g.row('Downsample') + .addPanel( + g.panel( + 'Rate', + 'Shows rate of execution for downsampling against blocks that are stored in the bucket by compaction group.' + ) + + g.queryPanel( + 'sum(rate(thanos_compact_downsample_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, group)', + 'downsample {{job}} {{group}}' + ) + + g.stack + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of executed downsampling against blocks that are stored in the bucket.') + + g.qpsErrTotalPanel( + 'thanos_compact_downsample_failed_total{namespace="$namespace",job=~"$job"}', + 'thanos_compact_downsample_total{namespace="$namespace",job=~"$job"}', + ) + ) + ) + .addRow( + g.row('Garbage Collection') + .addPanel( + g.panel( + 'Rate', + 'Shows rate of execution for removals of blocks if their data is available as part of a block with a higher compaction level.' + ) + + g.queryPanel( + 'sum(rate(thanos_compact_garbage_collection_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)', + 'garbage collection {{job}}' + ) + + g.stack + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of executed garbage collections.') + + g.qpsErrTotalPanel( + 'thanos_compact_garbage_collection_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_compact_garbage_collection_total{namespace="$namespace",job=~"$job"}', + ) + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to execute garbage collection in quantiles.') + + g.latencyPanel('thanos_compact_garbage_collection_duration_seconds', 'namespace="$namespace",job=~"$job"') + ) + ) + .addRow( + g.row('Sync Meta') + .addPanel( + g.panel( + 'Rate', + 'Shows rate of execution for all meta files from blocks in the bucket into the memory.' + ) + + g.queryPanel( + 'sum(rate(thanos_compact_sync_meta_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)', + 'sync {{job}}' + ) + + g.stack + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of executed meta file sync.') + + g.qpsErrTotalPanel( + 'thanos_compact_sync_meta_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_compact_sync_meta_total{namespace="$namespace",job=~"$job"}', + ) + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to execute meta file sync, in quantiles.') + + g.latencyPanel('thanos_compact_sync_meta_duration_seconds', 'namespace="$namespace",job=~"$job"') + ) + ) + .addRow( + g.row('Object Store Operations') + .addPanel( + g.panel('Rate', 'Shows rate of execution for operations against the bucket.') + + g.queryPanel( + 'sum(rate(thanos_objstore_bucket_operations_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, operation)', + '{{job}} {{operation}}' + ) + + g.stack + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of executed operations against the bucket.') + + g.qpsErrTotalPanel( + 'thanos_objstore_bucket_operation_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_objstore_bucket_operations_total{namespace="$namespace",job=~"$job"}', + ) + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to execute operations against the bucket, in quantiles.') + + g.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', 'namespace="$namespace",job=~"$job"') + ) + ) + .addRow( + g.resourceUtilizationRow() + ) + + g.template('namespace', thanos.dashboard.namespaceQuery) + + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.compactor, true, '%(jobPrefix)s.*' % thanos.compactor) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.compactor, true, '.*'), + + __overviewRows__+:: [ + g.row('Compact') + .addPanel( + g.panel( + 'Compaction Rate', + 'Shows rate of execution for compactions against blocks that are stored in the bucket by compaction group.' + ) + + g.queryPanel( + 'sum(rate(thanos_compact_group_compactions_total{namespace="$namespace",%(selector)s}[$interval])) by (job)' % thanos.compactor, + 'compaction {{job}}' + ) + + g.stack + + g.addDashboardLink(thanos.compactor.title) + ) + .addPanel( + g.panel( + 'Compaction Errors', + 'Shows ratio of errors compared to the total number of executed compactions against blocks that are stored in the bucket.' + ) + + g.qpsErrTotalPanel( + 'thanos_compact_group_compactions_failures_total{namespace="$namespace",%(selector)s}' % thanos.compactor, + 'thanos_compact_group_compactions_total{namespace="$namespace",%(selector)s}' % thanos.compactor, + ) + + g.addDashboardLink(thanos.compactor.title) + ) + + g.collapse, + ], + }, +} diff --git a/mixin/thanos/dashboards/dashboards.libsonnet b/mixin/thanos/dashboards/dashboards.libsonnet new file mode 100644 index 0000000000..f12f527103 --- /dev/null +++ b/mixin/thanos/dashboards/dashboards.libsonnet @@ -0,0 +1,8 @@ +(import 'querier.libsonnet') + +(import 'store.libsonnet') + +(import 'sidecar.libsonnet') + +(import 'receiver.libsonnet') + +(import 'ruler.libsonnet') + +(import 'compactor.libsonnet') + +(import 'overview.libsonnet') + +(import 'defaults.libsonnet') diff --git a/mixin/thanos/dashboards/defaults.libsonnet b/mixin/thanos/dashboards/defaults.libsonnet new file mode 100644 index 0000000000..510baf4ba7 --- /dev/null +++ b/mixin/thanos/dashboards/defaults.libsonnet @@ -0,0 +1,49 @@ +{ + local thanos = self, + local grafanaDashboards = super.grafanaDashboards, + local grafana = import 'grafonnet/grafana.libsonnet', + local template = grafana.template, + + dashboard:: { + prefix: 'Thanos / ', + tags: error 'must provide dashboard tags', + namespaceQuery: error 'must provide a query for namespace variable for dashboard template', + }, + + // Automatically add a uid to each dashboard based on the base64 encoding + // of the file name and set the timezone to be 'default'. + grafanaDashboards:: { + [filename]: grafanaDashboards[filename] { + uid: std.md5(filename), + timezone: '', + tags: thanos.dashboard.tags, + + // Modify tooltip to only show a single value + rows: [ + row { + panels: [ + panel { + tooltip+: { + shared: false, + }, + } + for panel in super.panels + ], + } + for row in super.rows + ], + + templating+: { + list+: [ + template.interval( + 'interval', + '5m,10m,30m,1h,6h,12h,auto', + label='interval', + current='5m', + ), + ], + }, + } + for filename in std.objectFields(grafanaDashboards) + }, +} diff --git a/mixin/thanos/dashboards/overview.libsonnet b/mixin/thanos/dashboards/overview.libsonnet new file mode 100644 index 0000000000..ad5328676e --- /dev/null +++ b/mixin/thanos/dashboards/overview.libsonnet @@ -0,0 +1,35 @@ +local g = import '../thanos-grafana-builder/builder.libsonnet'; + +{ + local thanos = self, + overview:: { + title: error 'must provide title for Thanos Overview dashboard', + }, + grafanaDashboards+:: { + 'overview.json': + g.dashboard(thanos.overview.title) + + g.template('namespace', thanos.dashboard.namespaceQuery), + }, +} + +{ + local grafanaDashboards = super.grafanaDashboards, + grafanaDashboards+:: { + 'overview.json'+: { + + __enumeratedRows__+:: std.foldl( + function(acc, row) + local n = std.length(row.panels); + local panelIndex = acc.counter; + local panels = std.makeArray( + n, function(i) + row.panels[i] { id: panelIndex + i } + ); + acc { counter:: acc.counter + n, rows+: [row { panels: panels }] }, + grafanaDashboards.__overviewRows__, + { counter:: 1, rows: [] } + ), + + rows+: self.__enumeratedRows__.rows, + }, + }, +} diff --git a/mixin/thanos/dashboards/querier.libsonnet b/mixin/thanos/dashboards/querier.libsonnet new file mode 100644 index 0000000000..a6fd758b23 --- /dev/null +++ b/mixin/thanos/dashboards/querier.libsonnet @@ -0,0 +1,193 @@ +local g = import '../thanos-grafana-builder/builder.libsonnet'; + +{ + local thanos = self, + querier+:: { + jobPrefix: error 'must provide job prefix for Thanos Query dashboard', + selector: error 'must provide selector for Thanos Query dashboard', + title: error 'must provide title for Thanos Query dashboard', + }, + grafanaDashboards+:: { + 'querier.json': + g.dashboard(thanos.querier.title) + .addRow( + g.row('Instant Query API') + .addPanel( + g.panel('Rate', 'Shows rate of requests against /query for the given time.') + + g.httpQpsPanel('http_requests_total', 'namespace="$namespace",job=~"$job",handler="query"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the the total number of handled requests against /query.') + + g.httpErrPanel('http_requests_total', 'namespace="$namespace",job=~"$job",handler="query"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests in quantiles.') + + g.latencyPanel('http_request_duration_seconds', 'namespace="$namespace",job=~"$job",handler="query"') + ) + ) + .addRow( + g.row('Range Query API') + .addPanel( + g.panel('Rate', 'Shows rate of requests against /query_range for the given time range.') + + g.httpQpsPanel('http_requests_total', 'namespace="$namespace",job=~"$job",handler="query_range"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the the total number of handled requests against /query_range.') + + g.httpErrPanel('http_requests_total', 'namespace="$namespace",job=~"$job",handler="query_range"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests in quantiles.') + + g.latencyPanel('http_request_duration_seconds', 'namespace="$namespace",job=~"$job",handler="query_range"') + ) + ) + .addRow( + g.row('Query Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of requests against /query for the given time, with handlers and codes.') + + g.httpQpsPanelDetailed('http_requests_total', 'namespace="$namespace",job=~"$job"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the the total number of handled requests, in more detail.') + + g.httpErrDetailsPanel('http_requests_total', 'namespace="$namespace",job=~"$job"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests in quantiles.') + + g.httpLatencyDetailsPanel('http_request_duration_seconds', 'namespace="$namespace",job=~"$job"') + ) + + g.collapse + ) + .addRow( + g.row('gRPC (Unary)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests from other queriers.') + + g.grpcQpsPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the the total number of handled requests from other queriers.') + + g.grpcErrorsPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from other queriers, in quantiles.') + + g.grpcLatencyPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests, with grpc methods and codes from other queriers.') + + g.grpcQpsPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the the total number of handled requests from other queriers.') + + g.grpcErrDetailsPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from other queriers, in quantiles.') + + g.grpcLatencyPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + + g.collapse + ) + .addRow( + g.row('gRPC (Stream)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests from other queriers.') + + g.grpcQpsPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the the total number of handled requests from other queriers.') + + g.grpcErrorsPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from other queriers, in quantiles') + + g.grpcLatencyPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests, with grpc methods and codes.') + + g.grpcQpsPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the the total number of handled requests.') + + g.grpcErrDetailsPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests in quantiles.') + + g.grpcLatencyPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + + g.collapse + ) + .addRow( + g.row('DNS') + .addPanel( + g.panel('Rate', 'Shows rate of DNS lookups to discover stores.') + + g.queryPanel( + 'sum(rate(thanos_querier_store_apis_dns_lookups_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)', + 'lookups {{job}}' + ) + ) + .addPanel( + g.panel('Errors', 'Shows ratio of failures compared to the the total number of executed DNS lookups.') + + g.qpsErrTotalPanel( + 'thanos_querier_store_apis_dns_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_querier_store_apis_dns_lookups_total{namespace="$namespace",job=~"$job"}', + ) + ) + ) + .addRow( + g.resourceUtilizationRow() + ) + + g.template('namespace', thanos.dashboard.namespaceQuery) + + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.querier, true, '%(jobPrefix)s.*' % thanos.querier) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.querier, true, '.*'), + + __overviewRows__+:: [ + g.row('Instant Query') + .addPanel( + g.panel('Requests Rate', 'Shows rate of requests against /query for the given time.') + + g.httpQpsPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query"' % thanos.querier) + + g.addDashboardLink(thanos.querier.title) + ) + .addPanel( + g.panel('Requests Errors', 'Shows ratio of errors compared to the the total number of handled requests against /query.') + + g.httpErrPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query"' % thanos.querier) + + g.addDashboardLink(thanos.querier.title) + ) + .addPanel( + g.sloLatency( + 'Latency 99th Percentile', + 'Shows how long has it taken to handle requests.', + 'http_request_duration_seconds_bucket{namespace="$namespace",%(selector)s,handler="query"}' % thanos.querier, + 0.99, + 0.5, + 1 + ) + + g.addDashboardLink(thanos.querier.title) + ), + + g.row('Range Query') + .addPanel( + g.panel('Requests Rate', 'Shows rate of requests against /query_range for the given time range.') + + g.httpQpsPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query_range"' % thanos.querier) + + g.addDashboardLink(thanos.querier.title) + ) + .addPanel( + g.panel('Requests Errors', 'Shows ratio of errors compared to the the total number of handled requests against /query_range.') + + g.httpErrPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query_range"' % thanos.querier) + + g.addDashboardLink(thanos.querier.title) + ) + .addPanel( + g.sloLatency( + 'Latency 99th Percentile', + 'Shows how long has it taken to handle requests.', + 'http_request_duration_seconds_bucket{namespace="$namespace",%(selector)s,handler="query_range"}' % thanos.querier, + 0.99, + 0.5, + 1 + ) + + g.addDashboardLink(thanos.querier.title) + ), + ], + }, +} diff --git a/mixin/thanos/dashboards/receiver.libsonnet b/mixin/thanos/dashboards/receiver.libsonnet new file mode 100644 index 0000000000..3dddc94e76 --- /dev/null +++ b/mixin/thanos/dashboards/receiver.libsonnet @@ -0,0 +1,171 @@ +local g = import '../thanos-grafana-builder/builder.libsonnet'; + +{ + local thanos = self, + receiver+:: { + jobPrefix: error 'must provide job prefix for Thanos Receive dashboard', + selector: error 'must provide selector for Thanos Receive dashboard', + title: error 'must provide title for Thanos Receive dashboard', + }, + grafanaDashboards+:: { + 'receiver.json': + g.dashboard(thanos.receiver.title) + .addRow( + g.row('Incoming Request') + .addPanel( + g.panel('Rate', 'Shows rate of incoming requests.') + + g.httpQpsPanel('http_requests_total', 'handler="receive",namespace="$namespace",job=~"$job"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled incoming requests.') + + g.httpErrPanel('http_requests_total', 'handler="receive",namespace="$namespace",job=~"$job"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle incoming requests in quantiles.') + + g.latencyPanel('http_request_duration_seconds', 'handler="receive",namespace="$namespace",job=~"$job"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of incoming requests.') + + g.httpQpsPanelDetailed('http_requests_total', 'handler="receive",namespace="$namespace",job=~"$job"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled incoming requests.') + + g.httpErrDetailsPanel('http_requests_total', 'handler="receive",namespace="$namespace",job=~"$job"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle incoming requests in quantiles.') + + g.httpLatencyDetailsPanel('http_request_duration_seconds', 'handler="receive",namespace="$namespace",job=~"$job"') + ) + + g.collapse + ) + .addRow( + g.row('Forward Request') + .addPanel( + g.panel('Rate', 'Shows rate of forwarded requests to other receive nodes.') + + g.queryPanel( + 'sum(rate(thanos_receive_forward_requests_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)', + 'all {{job}}', + ) + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of forwareded requests to other receive nodes.') + + g.qpsErrTotalPanel( + 'thanos_receive_forward_requests_total{namespace="$namespace",job=~"$job",result="error"}', + 'thanos_receive_forward_requests_total{namespace="$namespace",job=~"$job"}', + ) + ) + ) + .addRow( + g.row('gRPC (Unary)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests from queriers.') + + g.grpcQpsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrorsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests from queriers.') + + g.grpcQpsPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrDetailsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + + g.collapse + ) + .addRow( + g.row('gRPC (Stream)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests from queriers.') + + g.grpcQpsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrorsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests from queriers.') + + g.grpcQpsPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrDetailsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + + g.collapse + ) + .addRow( + g.row('Last Updated') + .addPanel( + g.panel('Successful Upload', 'Shows the relative time of last successful upload to the object-store bucket.') + + g.tablePanel( + ['time() - max(thanos_objstore_bucket_last_successful_upload_time{namespace="$namespace",job=~"$job"}) by (job, bucket)'], + { + Value: { + alias: 'Uploaded Ago', + unit: 's', + type: 'number', + }, + }, + ) + ) + ) + .addRow( + g.resourceUtilizationRow() + ) + + g.template('namespace', thanos.dashboard.namespaceQuery) + + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.receiver, true, '%(jobPrefix)s.*' % thanos.receiver) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.receiver, true, '.*'), + + __overviewRows__+:: [ + g.row('Receive') + .addPanel( + g.panel('Incoming Requests Rate', 'Shows rate of incoming requests.') + + g.httpQpsPanel('http_requests_total', 'handler="receive",namespace="$namespace",%(selector)s' % thanos.receiver) + + g.addDashboardLink(thanos.receiver.title) + ) + .addPanel( + g.panel('Incoming Requests Errors', 'Shows ratio of errors compared to the total number of handled incoming requests.') + + g.httpErrPanel('http_requests_total', 'handler="receive",namespace="$namespace",%(selector)s' % thanos.receiver) + + g.addDashboardLink(thanos.receiver.title) + ) + .addPanel( + g.sloLatency( + 'Incoming Requests Latency 99th Percentile', + 'Shows how long has it taken to handle incoming requests.', + 'http_request_duration_seconds_bucket{handler="receive",namespace="$namespace",%(selector)s}' % thanos.receiver, + 0.99, + 0.5, + 1 + ) + + g.addDashboardLink(thanos.receiver.title) + ), + ], + }, +} diff --git a/mixin/thanos/dashboards/ruler.libsonnet b/mixin/thanos/dashboards/ruler.libsonnet new file mode 100644 index 0000000000..7c03a88658 --- /dev/null +++ b/mixin/thanos/dashboards/ruler.libsonnet @@ -0,0 +1,144 @@ +local g = import '../thanos-grafana-builder/builder.libsonnet'; + +{ + local thanos = self, + ruler+:: { + jobPrefix: error 'must provide job prefix for Thanos Ruler dashboard', + selector: error 'must provide selector for Thanos Ruler dashboard', + title: error 'must provide title for Thanos Ruler dashboard', + }, + grafanaDashboards+:: { + 'ruler.json': + g.dashboard(thanos.ruler.title) + .addRow( + g.row('Alert Sent') + .addPanel( + g.panel('Dropped Rate', 'Shows rate of dropped alerts.') + + g.queryPanel( + 'sum(rate(thanos_alert_sender_alerts_dropped_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, alertmanager)', + '{{job}} {{alertmanager}}' + ) + ) + .addPanel( + g.panel('Sent Rate', 'Shows rate of alerts that successfully sent to alert manager.') + + g.queryPanel( + 'sum(rate(thanos_alert_sender_alerts_sent_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, alertmanager)', + '{{job}} {{alertmanager}}' + ) + + g.stack + ) + .addPanel( + g.panel('Sent Errors', 'Shows ratio of errors compared to the total number of sent alerts.') + + g.qpsErrTotalPanel( + 'thanos_alert_sender_errors_total{namespace="$namespace",job=~"$job"}', + 'thanos_alert_sender_alerts_sent_total{namespace="$namespace",job=~"$job"}', + ) + ) + .addPanel( + g.panel('Sent Duration', 'Shows how long has it taken to send alerts to alert manager.') + + g.latencyPanel('thanos_alert_sender_latency_seconds', 'namespace="$namespace",job=~"$job"'), + ) + ) + .addRow( + g.row('gRPC (Unary)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests.') + + g.grpcQpsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests.') + + g.grpcErrorsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests, in quantiles.') + + g.grpcLatencyPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests.') + + g.grpcQpsPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests.') + + g.grpcErrDetailsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests, in quantiles.') + + g.grpcLatencyPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + + g.collapse + ) + .addRow( + g.row('gRPC (Stream)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests.') + + g.grpcQpsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests.') + + g.grpcErrorsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests, in quantiles') + + g.grpcLatencyPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests.') + + g.grpcQpsPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests.') + + g.grpcErrDetailsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests, in quantiles') + + g.grpcLatencyPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + + g.collapse + ) + .addRow( + g.resourceUtilizationRow() + ) + + g.template('namespace', thanos.dashboard.namespaceQuery) + + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.ruler, true, '%(jobPrefix)s.*' % thanos.ruler) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.ruler, true, '.*'), + + __overviewRows__+:: [ + g.row('Rule') + .addPanel( + g.panel('Alert Sent Rate', 'Shows rate of alerts that successfully sent to alert manager.') + + g.queryPanel( + 'sum(rate(thanos_alert_sender_alerts_sent_total{namespace="$namespace",%(selector)s}[$interval])) by (job, alertmanager)' % thanos.ruler, + '{{job}} {{alertmanager}}' + ) + + g.addDashboardLink(thanos.ruler.title) + + g.stack + ) + .addPanel( + g.panel('Alert Sent Errors', 'Shows ratio of errors compared to the total number of sent alerts.') + + g.qpsErrTotalPanel( + 'thanos_alert_sender_errors_total{namespace="$namespace",%(selector)s}' % thanos.ruler, + 'thanos_alert_sender_alerts_sent_total{namespace="$namespace",%(selector)s}' % thanos.ruler, + ) + + g.addDashboardLink(thanos.ruler.title) + ) + .addPanel( + g.sloLatency( + 'Alert Sent Duration', + 'Shows how long has it taken to send alerts to alert manager.', + 'thanos_alert_sender_latency_seconds_bucket{namespace="$namespace",%(selector)s}' % thanos.ruler, + 0.99, + 0.5, + 1 + ) + + g.addDashboardLink(thanos.ruler.title) + ) + + g.collapse, + ], + }, +} diff --git a/mixin/thanos/dashboards/sidecar.libsonnet b/mixin/thanos/dashboards/sidecar.libsonnet new file mode 100644 index 0000000000..ef6cae0cae --- /dev/null +++ b/mixin/thanos/dashboards/sidecar.libsonnet @@ -0,0 +1,145 @@ +local g = import '../thanos-grafana-builder/builder.libsonnet'; + +{ + local thanos = self, + sidecar+:: { + jobPrefix: error 'must provide job prefix for Thanos Sidecar dashboard', + selector: error 'must provide selector for Thanos Sidecar dashboard', + title: error 'must provide title for Thanos Sidecar dashboard', + }, + grafanaDashboards+:: { + 'sidecar.json': + g.dashboard(thanos.sidecar.title) + .addRow( + g.row('gRPC (Unary)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests from queriers.') + + g.grpcQpsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrorsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests from queriers.') + + g.grpcQpsPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrDetailsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + + g.collapse + ) + .addRow( + g.row('gRPC (Stream)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests from queriers.') + + g.grpcQpsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors') + + g.grpcErrorsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests from queriers.') + + g.grpcQpsPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrDetailsPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + + g.collapse + ) + .addRow( + g.row('Last Updated') + .addPanel( + g.panel('Successful Upload', 'Shows the relative time of last successful upload to the object-store bucket.') + + g.tablePanel( + ['time() - max(thanos_objstore_bucket_last_successful_upload_time{namespace="$namespace",job=~"$job"}) by (job, bucket)'], + { + Value: { + alias: 'Uploaded Ago', + unit: 's', + type: 'number', + }, + }, + ) + ) + ) + .addRow( + g.row('Bucket Operations') + .addPanel( + g.panel('Rate') + + g.queryPanel( + 'sum(rate(thanos_objstore_bucket_operations_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, operation)', + '{{job}} {{operation}}' + ) + + g.stack + ) + .addPanel( + g.panel('Errors') + + g.qpsErrTotalPanel( + 'thanos_objstore_bucket_operation_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_objstore_bucket_operations_total{namespace="$namespace",job=~"$job"}', + ) + ) + .addPanel( + g.panel('Duration') + + g.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', 'namespace="$namespace",job=~"$job"') + ) + ) + .addRow( + g.resourceUtilizationRow() + ) + + g.template('namespace', thanos.dashboard.namespaceQuery) + + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.sidecar, true, '%(jobPrefix)s.*' % thanos.sidecar) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.sidecar, true, '.*'), + + __overviewRows__+:: [ + g.row('Sidecar') + .addPanel( + g.panel('gPRC (Unary) Rate', 'Shows rate of handled Unary gRPC requests from queriers.') + + g.grpcQpsPanel('server', 'namespace="$namespace",%(selector)s,grpc_type="unary"' % thanos.sidecar) + + g.addDashboardLink(thanos.sidecar.title) + ) + .addPanel( + g.panel('gPRC (Unary) Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrorsPanel('server', 'namespace="$namespace",%(selector)s,grpc_type="unary"' % thanos.sidecar) + + g.addDashboardLink(thanos.sidecar.title) + ) + .addPanel( + g.sloLatency( + 'gPRC (Unary) Latency 99th Percentile', + 'Shows how long has it taken to handle requests from queriers, in quantiles.', + 'grpc_server_handling_seconds_bucket{grpc_type="unary",namespace="$namespace",%(selector)s}' % thanos.sidecar, + 0.99, + 0.5, + 1 + ) + + g.addDashboardLink(thanos.sidecar.title) + ), + ], + }, +} diff --git a/mixin/thanos/dashboards/store.libsonnet b/mixin/thanos/dashboards/store.libsonnet new file mode 100644 index 0000000000..e29ca992e2 --- /dev/null +++ b/mixin/thanos/dashboards/store.libsonnet @@ -0,0 +1,276 @@ +local g = import '../thanos-grafana-builder/builder.libsonnet'; + +{ + local thanos = self, + store+:: { + jobPrefix: error 'must provide job prefix for Thanos Store dashboard', + selector: error 'must provide selector for Thanos Store dashboard', + title: error 'must provide title for Thanos Store dashboard', + }, + grafanaDashboards+:: { + 'store.json': + g.dashboard(thanos.store.title) + .addRow( + g.row('gRPC (Unary)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests from queriers.') + + g.grpcQpsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrorsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Unary gRPC requests from queriers.') + + g.grpcQpsPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrDetailsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="unary"') + ) + + g.collapse + ) + .addRow( + g.row('gRPC (Stream)') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests from queriers.') + + g.grpcQpsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrorsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + ) + .addRow( + g.row('Detailed') + .addPanel( + g.panel('Rate', 'Shows rate of handled Streamed gRPC requests from queriers.') + + g.grpcQpsPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrDetailsPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + + g.grpcLatencyPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + ) + + g.collapse + ) + .addRow( + g.row('Bucket Operations') + .addPanel( + g.panel('Rate', 'Shows rate of execution for operations against the bucket.') + + g.queryPanel( + 'sum(rate(thanos_objstore_bucket_operations_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, operation)', + '{{job}} {{operation}}' + ) + + g.stack + ) + .addPanel( + g.panel('Errors', 'Shows ratio of errors compared to the total number of executed operations against the bucket.') + + g.qpsErrTotalPanel( + 'thanos_objstore_bucket_operation_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_objstore_bucket_operations_total{namespace="$namespace",job=~"$job"}', + ) + ) + .addPanel( + g.panel('Duration', 'Shows how long has it taken to execute operations against the bucket, in quantiles.') + + g.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', 'namespace="$namespace",job=~"$job"') + ) + ) + .addRow( + g.row('Block Operations') + .addPanel( + g.panel('Block Load Rate', 'Shows rate of block loads from the bucket.') + + g.queryPanel( + 'sum(rate(thanos_bucket_store_block_loads_total{namespace="$namespace",job=~"$job"}[$interval]))', + 'block loads' + ) + + g.stack + ) + .addPanel( + g.panel('Block Load Errors', 'Shows ratio of errors compared to the total number of block loads from the bucket.') + + g.qpsErrTotalPanel( + 'thanos_bucket_store_block_load_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_bucket_store_block_loads_total{namespace="$namespace",job=~"$job"}', + ) + ) + .addPanel( + g.panel('Block Drop Rate', 'Shows rate of block drops.') + + g.queryPanel( + 'sum(rate(thanos_bucket_store_block_drops_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, operation)', + 'block drops {{job}}' + ) + + g.stack + ) + .addPanel( + g.panel('Block Drop Errors', 'Shows ratio of errors compared to the total number of block drops.') + + g.qpsErrTotalPanel( + 'thanos_bucket_store_block_drop_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_bucket_store_block_drops_total{namespace="$namespace",job=~"$job"}', + ) + ) + ) + .addRow( + g.row('Cache Operations') + .addPanel( + g.panel('Requests', 'Show rate of cache requests.') + + g.queryPanel( + 'sum(rate(thanos_store_index_cache_requests_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, item_type)', + '{{job}} {{item_type}}', + ) + + g.stack + ) + .addPanel( + g.panel('Hits', 'Shows ratio of errors compared to the total number of cache hits.') + + g.queryPanel( + 'sum(rate(thanos_store_index_cache_hits_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, item_type)', + '{{job}} {{item_type}}', + ) + + g.stack + ) + .addPanel( + g.panel('Added', 'Show rate of added items to cache.') + + g.queryPanel( + 'sum(rate(thanos_store_index_cache_items_added_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, item_type)', + '{{job}} {{item_type}}', + ) + + g.stack + ) + .addPanel( + g.panel('Evicted', 'Show rate of evicted items from cache.') + + g.queryPanel( + 'sum(rate(thanos_store_index_cache_items_evicted_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, item_type)', + '{{job}} {{item_type}}', + ) + + g.stack + ) + ) + .addRow( + g.row('Store Sent') + .addPanel( + g.panel('Chunk Size', 'Shows size of chunks that have sent to the bucket, in bytes.') + + g.queryPanel( + [ + 'histogram_quantile(0.99, sum(rate(thanos_bucket_store_sent_chunk_size_bytes_bucket{namespace="$namespace",job=~"$job"}[$interval])) by (job, le))', + 'sum(rate(thanos_bucket_store_sent_chunk_size_bytes_sum{namespace="$namespace",job=~"$job"}[$interval])) by (job) / sum(rate(thanos_bucket_store_sent_chunk_size_bytes_count{namespace="$namespace",job=~"$job"}[$interval])) by (job)', + 'histogram_quantile(0.99, sum(rate(thanos_bucket_store_sent_chunk_size_bytes_bucket{namespace="$namespace",job=~"$job"}[$interval])) by (job, le))', + ], + [ + 'P99', + 'mean', + 'P50', + ], + ) + ) + + { yaxes: g.yaxes('decbytes') }, + ) + .addRow( + g.row('Series Operations') + .addPanel( + g.panel('Block queried') + + g.queryPanel( + [ + 'thanos_bucket_store_series_blocks_queried{namespace="$namespace",job=~"$job",quantile="0.99"}', + 'sum(rate(thanos_bucket_store_series_blocks_queried_sum{namespace="$namespace",job=~"$job"}[$interval])) by (job) / sum(rate(thanos_bucket_store_series_blocks_queried_count{namespace="$namespace",job=~"$job"}[$interval])) by (job)', + 'thanos_bucket_store_series_blocks_queried{namespace="$namespace",job=~"$job",quantile="0.50"}', + ], [ + 'P99', + 'mean {{job}}', + 'P50', + ], + ) + ) + .addPanel( + g.panel('Data Fetched') + + g.queryPanel( + [ + 'thanos_bucket_store_series_data_fetched{namespace="$namespace",job=~"$job",quantile="0.99"}', + 'sum(rate(thanos_bucket_store_series_data_fetched_sum{namespace="$namespace",job=~"$job"}[$interval])) by (job) / sum(rate(thanos_bucket_store_series_data_fetched_count{namespace="$namespace",job=~"$job"}[$interval])) by (job)', + 'thanos_bucket_store_series_data_fetched{namespace="$namespace",job=~"$job",quantile="0.50"}', + ], [ + 'P99', + 'mean {{job}}', + 'P50', + ], + ) + ) + .addPanel( + g.panel('Result series') + + g.queryPanel( + [ + 'thanos_bucket_store_series_result_series{namespace="$namespace",job=~"$job",quantile="0.99"}', + 'sum(rate(thanos_bucket_store_series_result_series_sum{namespace="$namespace",job=~"$job"}[$interval])) by (job) / sum(rate(thanos_bucket_store_series_result_series_count{namespace="$namespace",job=~"$job"}[$interval])) by (job)', + 'thanos_bucket_store_series_result_series{namespace="$namespace",job=~"$job",quantile="0.50"}', + ], [ + 'P99', + 'mean {{job}}', + 'P50', + ], + ) + ) + ) + .addRow( + g.row('Series Operation Durations') + .addPanel( + g.panel('Get All', 'Shows how long has it taken to get all series.') + + g.latencyPanel('thanos_bucket_store_series_get_all_duration_seconds', 'namespace="$namespace",job=~"$job"') + ) + .addPanel( + g.panel('Merge', 'Shows how long has it taken to merge series.') + + g.latencyPanel('thanos_bucket_store_series_merge_duration_seconds_bucket', 'namespace="$namespace",job=~"$job"') + ) + .addPanel( + g.panel('Gate', 'Shows how long has it taken for a series to wait at the gate.') + + g.latencyPanel('thanos_bucket_store_series_gate_duration_seconds_bucket', 'namespace="$namespace",job=~"$job"') + ) + ) + .addRow( + g.resourceUtilizationRow() + ) + + g.template('namespace', thanos.dashboard.namespaceQuery) + + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.store, true, '%(jobPrefix)s.*' % thanos.store) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.store, true, '.*'), + + __overviewRows__+:: [ + g.row('Store') + .addPanel( + g.panel('gPRC (Unary) Rate', 'Shows rate of handled Unary gRPC requests from queriers.') + + g.grpcQpsPanel('server', 'namespace="$namespace",%(selector)s,grpc_type="unary"' % thanos.store) + + g.addDashboardLink(thanos.store.title) + ) + .addPanel( + g.panel('gPRC (Unary) Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + + g.grpcErrorsPanel('server', 'namespace="$namespace",%(selector)s,grpc_type="unary"' % thanos.store) + + g.addDashboardLink(thanos.store.title) + ) + .addPanel( + g.sloLatency( + 'gRPC Latency 99th Percentile', + 'Shows how long has it taken to handle requests from queriers.', + 'grpc_server_handling_seconds_bucket{grpc_type="unary",namespace="$namespace",%(selector)s}' % thanos.store, + 0.99, + 0.5, + 1 + ) + + g.addDashboardLink(thanos.store.title) + ), + ], + }, +} diff --git a/mixin/thanos/defaults.libsonnet b/mixin/thanos/defaults.libsonnet new file mode 100644 index 0000000000..a6cf5e9cdc --- /dev/null +++ b/mixin/thanos/defaults.libsonnet @@ -0,0 +1,40 @@ +{ + querier+:: { + jobPrefix: 'thanos-querier', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sQuerier' % $.dashboard.prefix, + }, + store+:: { + jobPrefix: 'thanos-store', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sStore' % $.dashboard.prefix, + }, + receiver+:: { + jobPrefix: 'thanos-receiver', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sReceiver' % $.dashboard.prefix, + }, + ruler+:: { + jobPrefix: 'thanos-ruler', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sRuler' % $.dashboard.prefix, + }, + compactor+:: { + jobPrefix: 'thanos-compactor', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sCompactor' % $.dashboard.prefix, + }, + sidecar+:: { + jobPrefix: 'thanos-sidecar', + selector: 'job=~"%s.*"' % self.jobPrefix, + title: '%(prefix)sSidecar' % $.dashboard.prefix, + }, + overview+:: { + title: '%(prefix)sOverview' % $.dashboard.prefix, + }, + dashboard+:: { + prefix: 'Thanos / ', + tags: ['thanos-mixin'], + namespaceQuery: 'kube_pod_info', + }, +} diff --git a/mixin/thanos/mixin.libsonnet b/mixin/thanos/mixin.libsonnet new file mode 100644 index 0000000000..6590c396e4 --- /dev/null +++ b/mixin/thanos/mixin.libsonnet @@ -0,0 +1,3 @@ +(import 'dashboards/dashboards.libsonnet') + +(import 'alerts/alerts.libsonnet') + +(import 'rules/rules.libsonnet') diff --git a/mixin/thanos/rules.jsonnet b/mixin/thanos/rules.jsonnet new file mode 100644 index 0000000000..c50930e3b9 --- /dev/null +++ b/mixin/thanos/rules.jsonnet @@ -0,0 +1,4 @@ +( + (import 'mixin.libsonnet') + + (import 'defaults.libsonnet') +).prometheusRules diff --git a/mixin/thanos/rules/querier.libsonnet b/mixin/thanos/rules/querier.libsonnet new file mode 100644 index 0000000000..22706cf0e5 --- /dev/null +++ b/mixin/thanos/rules/querier.libsonnet @@ -0,0 +1,73 @@ +{ + local thanos = self, + querier+:: { + selector: error 'must provide selector for Thanos Query recording rules', + }, + prometheusRules+:: { + groups+: [ + { + name: 'thanos-querier.rules', + rules: [ + { + record: ':grpc_client_failures_per_unary:sum_rate', + expr: ||| + ( + sum(rate(grpc_client_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", %(selector)s, grpc_type="unary"}[5m])) + / + sum(rate(grpc_client_started_total{%(selector)s, grpc_type="unary"}[5m])) + ) + ||| % thanos.querier, + labels: { + }, + }, + { + record: ':grpc_client_failures_per_stream:sum_rate', + expr: ||| + ( + sum(rate(grpc_client_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", %(selector)s, grpc_type="server_stream"}[5m])) + / + sum(rate(grpc_client_started_total{%(selector)s, grpc_type="server_stream"}[5m])) + ) + ||| % thanos.querier, + labels: { + }, + }, + { + record: ':thanos_querier_store_apis_dns_failures_per_lookup:sum_rate', + expr: ||| + ( + sum(rate(thanos_querier_store_apis_dns_failures_total{%(selector)s}[5m])) + / + sum(rate(thanos_querier_store_apis_dns_lookups_total{%(selector)s}[5m])) + ) + ||| % thanos.querier, + labels: { + }, + }, + { + record: ':query_duration_seconds:histogram_quantile', + expr: ||| + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{%(selector)s, handler="query"}[5m])) by (le) + ) + ||| % thanos.querier, + labels: { + quantile: '0.99', + }, + }, + { + record: ':api_range_query_duration_seconds:histogram_quantile', + expr: ||| + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{%(selector)s, handler="query_range"}[5m])) by (le) + ) + ||| % thanos.querier, + labels: { + quantile: '0.99', + }, + }, + ], + }, + ], + }, +} diff --git a/mixin/thanos/rules/receiver.libsonnet b/mixin/thanos/rules/receiver.libsonnet new file mode 100644 index 0000000000..fc01667e58 --- /dev/null +++ b/mixin/thanos/rules/receiver.libsonnet @@ -0,0 +1,86 @@ +{ + local thanos = self, + receiver+:: { + selector: error 'must provide selector for Thanos Receive recording rules', + }, + prometheusRules+:: { + groups+: [ + { + name: 'thanos-receiver.rules', + rules: [ + { + record: ':grpc_server_failures_per_unary:sum_rate', + expr: ||| + sum( + rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", %(selector)s, grpc_type="unary"}[5m]) + / + rate(grpc_server_started_total{%(selector)s, grpc_type="unary"}[5m]) + ) + ||| % thanos.receiver, + labels: { + }, + }, + { + record: ':grpc_server_failures_per_stream:sum_rate', + expr: ||| + sum( + rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", %(selector)s, grpc_type="server_stream"}[5m]) + / + rate(grpc_server_started_total{%(selector)s, grpc_type="server_stream"}[5m]) + ) + ||| % thanos.receiver, + labels: { + }, + }, + { + record: ':http_failure_per_request:sum_rate', + expr: ||| + sum( + rate(http_requests_total{handler="receive", %(selector)s, code!~"5.."}[5m]) + / + rate(http_requests_total{handler="receive", %(selector)s}[5m]) + ) + ||| % thanos.receiver, + labels: { + }, + }, + { + record: ':http_request_duration_seconds:histogram_quantile', + expr: ||| + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{handler="receive", %(selector)s}[5m])) by (le) + ) + ||| % thanos.receiver, + labels: { + quantile: '0.99', + }, + }, + { + record: ':thanos_receive_forward_failure_per_requests:sum_rate', + expr: ||| + ( + sum(rate(thanos_receive_forward_requests_total{result="error", %(selector)s}[5m])) + / + sum(rate(thanos_receive_forward_requests_total{%(selector)s}[5m])) + ) + ||| % thanos.receiver, + labels: { + }, + }, + { + record: ':thanos_receive_hashring_file_failure_per_refresh:sum_rate', + expr: ||| + ( + sum(rate(thanos_receive_hashrings_file_errors_total{%(selector)s}[5m])) + / + sum(rate(thanos_receive_hashrings_file_refreshes_total{%(selector)s}[5m])) + ) + ||| % thanos.receiver, + labels: { + }, + }, + ], + }, + ], + }, +} diff --git a/mixin/thanos/rules/rules.libsonnet b/mixin/thanos/rules/rules.libsonnet new file mode 100644 index 0000000000..656b267b4f --- /dev/null +++ b/mixin/thanos/rules/rules.libsonnet @@ -0,0 +1,3 @@ +(import 'querier.libsonnet') + +(import 'receiver.libsonnet') + +(import 'store.libsonnet') diff --git a/mixin/thanos/rules/store.libsonnet b/mixin/thanos/rules/store.libsonnet new file mode 100644 index 0000000000..946df7bb16 --- /dev/null +++ b/mixin/thanos/rules/store.libsonnet @@ -0,0 +1,62 @@ +{ + local thanos = self, + store+:: { + selector: error 'must provide selector for Thanos Store recording rules', + }, + prometheusRules+:: { + groups+: [ + { + name: 'thanos-store.rules', + rules: [ + { + record: ':grpc_server_failures_per_unary:sum_rate', + expr: ||| + ( + sum(rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", %(selector)s, grpc_type="unary"}[5m])) + / + sum(rate(grpc_server_started_total{%(selector)s, grpc_type="unary"}[5m])) + ) + ||| % thanos.store, + labels: { + }, + }, + { + record: ':grpc_server_failures_per_stream:sum_rate', + expr: ||| + ( + sum(rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", %(selector)s, grpc_type="server_stream"}[5m])) + / + sum(rate(grpc_server_started_total{%(selector)s, grpc_type="server_stream"}[5m])) + ) + ||| % thanos.store, + labels: { + }, + }, + { + record: ':thanos_objstore_bucket_failures_per_operation:sum_rate', + expr: ||| + ( + sum(rate(thanos_objstore_bucket_operation_failures_total{%(selector)s}[5m])) + / + sum(rate(thanos_objstore_bucket_operations_total{%(selector)s}[5m])) + ) + ||| % thanos.store, + labels: { + }, + }, + { + record: ':thanos_objstore_bucket_operation_duration_seconds:histogram_quantile', + expr: ||| + histogram_quantile(0.99, + sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{%(selector)s}[5m])) by (le) + ) + ||| % thanos.store, + labels: { + quantile: '0.99', + }, + }, + ], + }, + ], + }, +} diff --git a/mixin/thanos/separated_alerts.jsonnet b/mixin/thanos/separated_alerts.jsonnet new file mode 100644 index 0000000000..79402a0b3f --- /dev/null +++ b/mixin/thanos/separated_alerts.jsonnet @@ -0,0 +1,7 @@ +{ + [group.name]: group + for group in ( + (import 'mixin.libsonnet') + + (import 'defaults.libsonnet') + ).prometheusAlerts.groups +} From a0e1771a5e25bf1b6994377e6f3a6ea1e13ab88d Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Fri, 13 Dec 2019 19:08:47 +0100 Subject: [PATCH 110/257] Fix broken doc links (#1889) Signed-off-by: Kemal Akkoyun --- docs/getting-started.md | 2 +- examples/dashboards/dashboards.md | 2 +- mixin/thanos/README.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index d1a46672e8..889d58bc61 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -91,7 +91,7 @@ If you want to add yourself to this list, let us know! ## Operating -See up to date [jsonnet mixins](https://github.com/thanos-io/thanos/tree/master/jsonnet/thanos-mixin) +See up to date [jsonnet mixins](https://github.com/thanos-io/thanos/tree/master/mixin/thanos/README.md) We also have example Grafana dashboards [here](/examples/dashboards/dashboards.md) and some [alerts](/examples/alerts/alerts.md) to get you started. ## Talks diff --git a/examples/dashboards/dashboards.md b/examples/dashboards/dashboards.md index 36f13fe703..df12f49bd5 100644 --- a/examples/dashboards/dashboards.md +++ b/examples/dashboards/dashboards.md @@ -15,4 +15,4 @@ These dashboards require Grafana 5 or above, importing them in older versions ar ## Configuration -All dashboards are generated using [`thanos-mixin`](../../jsonnet/thanos-mixin) and check out [README](../../jsonnet/thanos-mixin/README.md) for further information. +All dashboards are generated using [`thanos-mixin`](https://github.com/thanos-io/thanos/tree/master/mixin/thanos) and check out [README](https://github.com/thanos-io/thanos/tree/master/mixin/thanos/README.md) for further information. diff --git a/mixin/thanos/README.md b/mixin/thanos/README.md index 1472b88db8..f5aef3e212 100644 --- a/mixin/thanos/README.md +++ b/mixin/thanos/README.md @@ -35,7 +35,7 @@ To install: go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb ``` -> An e.g. of how to install a given version of this library: `jb install github.com/thanos-io/thanos/jsonnet/thanos-mixin@master`. +> An e.g. of how to install a given version of this library: `jb install github.com/thanos-io/thanos/mixin/thanos@master`. ## Use as a library @@ -44,7 +44,7 @@ To use the `thanos-mixin` as a dependency, simply use the `jsonnet-bundler` inst $ mkdir thanos-mixin; cd thanos-mixin $ jb init # Creates the initial/empty `jsonnetfile.json` # Install the thanos-mixin dependency -$ jb install github.com/thanos-io/thanos/jsonnet/thanos-mixin@master # Creates `vendor/` & `jsonnetfile.lock.json`, and fills in `jsonnetfile.json` +$ jb install github.com/thanos-io/thanos/mixin/thanos@master # Creates `vendor/` & `jsonnetfile.lock.json`, and fills in `jsonnetfile.json` ``` To update the `thanos-mixin` as a dependency, simply use the `jsonnet-bundler` update functionality: From 3b2573fb059ef8853dd2e6135a3b7455b6fa84f8 Mon Sep 17 00:00:00 2001 From: johncming Date: Sat, 14 Dec 2019 17:19:40 +0800 Subject: [PATCH 111/257] deduplicate addresses. (#1887) Signed-off-by: johncming --- pkg/discovery/cache/cache.go | 13 +++++++++-- pkg/discovery/cache/cache_test.go | 39 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 pkg/discovery/cache/cache_test.go diff --git a/pkg/discovery/cache/cache.go b/pkg/discovery/cache/cache.go index 22b5feb5ea..c619eaa793 100644 --- a/pkg/discovery/cache/cache.go +++ b/pkg/discovery/cache/cache.go @@ -38,12 +38,21 @@ func (c *Cache) Update(tgs []*targetgroup.Group) { // Addresses returns all the addresses from all target groups present in the Cache. func (c *Cache) Addresses() []string { + var addresses []string + var unique map[string]struct{} + c.Lock() defer c.Unlock() - var addresses []string + + unique = make(map[string]struct{}) for _, group := range c.tgs { for _, target := range group.Targets { - addresses = append(addresses, string(target[model.AddressLabel])) + addr := string(target[model.AddressLabel]) + if _, ok := unique[addr]; ok { + continue + } + addresses = append(addresses, addr) + unique[addr] = struct{}{} } } return addresses diff --git a/pkg/discovery/cache/cache_test.go b/pkg/discovery/cache/cache_test.go new file mode 100644 index 0000000000..cd40de760a --- /dev/null +++ b/pkg/discovery/cache/cache_test.go @@ -0,0 +1,39 @@ +package cache + +import ( + "reflect" + "testing" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +func TestCacheAddresses(t *testing.T) { + tgs := make(map[string]*targetgroup.Group) + tgs["g1"] = &targetgroup.Group{ + Targets: []model.LabelSet{ + model.LabelSet{model.AddressLabel: "localhost:9090"}, + model.LabelSet{model.AddressLabel: "localhost:9091"}, + model.LabelSet{model.AddressLabel: "localhost:9092"}, + }, + } + tgs["g2"] = &targetgroup.Group{ + Targets: []model.LabelSet{ + model.LabelSet{model.AddressLabel: "localhost:9091"}, + model.LabelSet{model.AddressLabel: "localhost:9092"}, + model.LabelSet{model.AddressLabel: "localhost:9093"}, + }, + } + + c := &Cache{tgs: tgs} + + expected := []string{ + "localhost:9090", + "localhost:9091", + "localhost:9092", + "localhost:9093", + } + if got := c.Addresses(); !reflect.DeepEqual(got, expected) { + t.Errorf("expected %v, want %v", got, expected) + } +} From bec866667bc5e500cb609a2b107c57af99e7b777 Mon Sep 17 00:00:00 2001 From: johncming Date: Sat, 14 Dec 2019 20:17:43 +0800 Subject: [PATCH 112/257] pkg/store: add http header. (#1893) Signed-off-by: johncming --- pkg/store/prometheus.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 6b9be0a6a9..42fa3f02e3 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -21,6 +21,7 @@ import ( "github.com/golang/snappy" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" + "github.com/prometheus/common/version" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage/remote" @@ -41,6 +42,8 @@ var statusToCode = map[int]codes.Code{ http.StatusInternalServerError: codes.Internal, } +var userAgent = fmt.Sprintf("Thanos/%s", version.Version) + // PrometheusStore implements the store node API on top of the Prometheus remote read API. type PrometheusStore struct { logger log.Logger @@ -399,6 +402,7 @@ func (p *PrometheusStore) startPromSeries(ctx context.Context, q *prompb.Query) } preq.Header.Add("Content-Encoding", "snappy") preq.Header.Set("Content-Type", "application/x-stream-protobuf") + preq.Header.Set("User-Agent", userAgent) spanReqDo, ctx := tracing.StartSpan(ctx, "query_prometheus_request") preq = preq.WithContext(ctx) presp, err := p.client.Do(preq) @@ -507,6 +511,7 @@ func (p *PrometheusStore) LabelNames(ctx context.Context, _ *storepb.LabelNamesR if err != nil { return nil, status.Error(codes.Internal, err.Error()) } + req.Header.Set("User-Agent", userAgent) span, ctx := tracing.StartSpan(ctx, "/prom_label_names HTTP[client]") defer span.Finish() @@ -567,6 +572,7 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue if err != nil { return nil, status.Error(codes.Internal, err.Error()) } + req.Header.Set("User-Agent", userAgent) span, ctx := tracing.StartSpan(ctx, "/prom_label_values HTTP[client]") defer span.Finish() From 2f29975982e25632a3eaccaf47d2401dcbfeef4b Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Mon, 16 Dec 2019 17:25:34 +0800 Subject: [PATCH 113/257] Clean up white noise and format all codes (#1876) Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/components/compact.md | 2 +- docs/quick-tutorial.md | 2 +- docs/service-discovery.md | 2 +- pkg/objstore/oss/oss.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/components/compact.md b/docs/components/compact.md index 30e536ed37..070d82b7b0 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -9,7 +9,7 @@ menu: components The compactor component of Thanos applies the compaction procedure of the Prometheus 2.0 storage engine to block data stored in object storage. It is generally not semantically concurrency safe and must be deployed as a singleton against a bucket. -It is also responsible for downsampling of data: +It is also responsible for downsampling of data: * creating 5m downsampling for blocks larger than **40 hours** (2d, 2w) * creating 1h downsampling for blocks larger than **10 days** (2w). diff --git a/docs/quick-tutorial.md b/docs/quick-tutorial.md index 9ed77ac59b..78a12d786f 100644 --- a/docs/quick-tutorial.md +++ b/docs/quick-tutorial.md @@ -95,7 +95,7 @@ thanos sidecar \ ### Uploading old metrics. -When sidecar is run with the `--shipper.upload-compacted` flag it will sync all older existing blocks from the Prometheus local storage on startup. +When sidecar is run with the `--shipper.upload-compacted` flag it will sync all older existing blocks from the Prometheus local storage on startup. NOTE: This assumes you never run sidecar with block uploading against this bucket. Otherwise manual steps are needed to remove overlapping blocks from the bucket. Those will be suggested by the sidecar verification process. diff --git a/docs/service-discovery.md b/docs/service-discovery.md index 23bd3ca899..cdb44e8493 100644 --- a/docs/service-discovery.md +++ b/docs/service-discovery.md @@ -13,7 +13,7 @@ SD is currently used in the following places within Thanos: * `Thanos Query` needs to know about [StoreAPI](https://github.com/thanos-io/thanos/blob/d3fb337da94d11c78151504b1fccb1d7e036f394/pkg/store/storepb/rpc.proto#L14) servers in order to query metrics from them. * `Thanos Rule` needs to know about `QueryAPI` servers in order to evaluate recording and alerting rules. -* `Thanos Rule` needs to know about `Alertmanagers` HA replicas in order to send alerts; only static option with DNS discovery. +* `Thanos Rule` needs to know about `Alertmanagers` HA replicas in order to send alerts; only static option with DNS discovery. There are currently several ways to configure SD, described below in more detail: diff --git a/pkg/objstore/oss/oss.go b/pkg/objstore/oss/oss.go index e4b2b14224..ff5ab1831f 100644 --- a/pkg/objstore/oss/oss.go +++ b/pkg/objstore/oss/oss.go @@ -139,7 +139,7 @@ func (b *Bucket) Delete(ctx context.Context, name string) error { // ObjectSize returns the size of the specified object. func (b *Bucket) ObjectSize(ctx context.Context, name string) (uint64, error) { - // https://github.com/aliyun/aliyun-oss-go-sdk/blob/cee409f5b4d75d7ad077cacb7e6f4590a7f2e172/oss/bucket.go#L668 + // refer to https://github.com/aliyun/aliyun-oss-go-sdk/blob/cee409f5b4d75d7ad077cacb7e6f4590a7f2e172/oss/bucket.go#L668. m, err := b.bucket.GetObjectMeta(name) if err != nil { return 0, err From ed9355cd207704b8ae9ae30be6e9c7f1519076db Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Mon, 16 Dec 2019 14:09:55 +0100 Subject: [PATCH 114/257] Fix jsonnet transitive dependency issue (#1898) Signed-off-by: Kemal Akkoyun --- mixin/thanos/dashboards/compactor.libsonnet | 2 +- mixin/thanos/dashboards/overview.libsonnet | 2 +- mixin/thanos/dashboards/querier.libsonnet | 2 +- mixin/thanos/dashboards/receiver.libsonnet | 2 +- mixin/thanos/dashboards/ruler.libsonnet | 2 +- mixin/thanos/dashboards/sidecar.libsonnet | 2 +- mixin/thanos/dashboards/store.libsonnet | 2 +- mixin/thanos/lib/thanos-grafana-builder/README.md | 6 ++++++ .../lib}/thanos-grafana-builder/builder.libsonnet | 0 .../{ => thanos/lib}/thanos-grafana-builder/grpc.libsonnet | 0 .../{ => thanos/lib}/thanos-grafana-builder/http.libsonnet | 0 mixin/{ => thanos/lib}/thanos-grafana-builder/slo.libsonnet | 0 12 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 mixin/thanos/lib/thanos-grafana-builder/README.md rename mixin/{ => thanos/lib}/thanos-grafana-builder/builder.libsonnet (100%) rename mixin/{ => thanos/lib}/thanos-grafana-builder/grpc.libsonnet (100%) rename mixin/{ => thanos/lib}/thanos-grafana-builder/http.libsonnet (100%) rename mixin/{ => thanos/lib}/thanos-grafana-builder/slo.libsonnet (100%) diff --git a/mixin/thanos/dashboards/compactor.libsonnet b/mixin/thanos/dashboards/compactor.libsonnet index 79f594395a..ff45c6dbf2 100644 --- a/mixin/thanos/dashboards/compactor.libsonnet +++ b/mixin/thanos/dashboards/compactor.libsonnet @@ -1,4 +1,4 @@ -local g = import '../thanos-grafana-builder/builder.libsonnet'; +local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, diff --git a/mixin/thanos/dashboards/overview.libsonnet b/mixin/thanos/dashboards/overview.libsonnet index ad5328676e..a0b1082a9d 100644 --- a/mixin/thanos/dashboards/overview.libsonnet +++ b/mixin/thanos/dashboards/overview.libsonnet @@ -1,4 +1,4 @@ -local g = import '../thanos-grafana-builder/builder.libsonnet'; +local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, diff --git a/mixin/thanos/dashboards/querier.libsonnet b/mixin/thanos/dashboards/querier.libsonnet index a6fd758b23..18ef95c064 100644 --- a/mixin/thanos/dashboards/querier.libsonnet +++ b/mixin/thanos/dashboards/querier.libsonnet @@ -1,4 +1,4 @@ -local g = import '../thanos-grafana-builder/builder.libsonnet'; +local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, diff --git a/mixin/thanos/dashboards/receiver.libsonnet b/mixin/thanos/dashboards/receiver.libsonnet index 3dddc94e76..dd35a6ee9f 100644 --- a/mixin/thanos/dashboards/receiver.libsonnet +++ b/mixin/thanos/dashboards/receiver.libsonnet @@ -1,4 +1,4 @@ -local g = import '../thanos-grafana-builder/builder.libsonnet'; +local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, diff --git a/mixin/thanos/dashboards/ruler.libsonnet b/mixin/thanos/dashboards/ruler.libsonnet index 7c03a88658..941c38b417 100644 --- a/mixin/thanos/dashboards/ruler.libsonnet +++ b/mixin/thanos/dashboards/ruler.libsonnet @@ -1,4 +1,4 @@ -local g = import '../thanos-grafana-builder/builder.libsonnet'; +local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, diff --git a/mixin/thanos/dashboards/sidecar.libsonnet b/mixin/thanos/dashboards/sidecar.libsonnet index ef6cae0cae..37878c7b11 100644 --- a/mixin/thanos/dashboards/sidecar.libsonnet +++ b/mixin/thanos/dashboards/sidecar.libsonnet @@ -1,4 +1,4 @@ -local g = import '../thanos-grafana-builder/builder.libsonnet'; +local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, diff --git a/mixin/thanos/dashboards/store.libsonnet b/mixin/thanos/dashboards/store.libsonnet index e29ca992e2..2f8d27d219 100644 --- a/mixin/thanos/dashboards/store.libsonnet +++ b/mixin/thanos/dashboards/store.libsonnet @@ -1,4 +1,4 @@ -local g = import '../thanos-grafana-builder/builder.libsonnet'; +local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, diff --git a/mixin/thanos/lib/thanos-grafana-builder/README.md b/mixin/thanos/lib/thanos-grafana-builder/README.md new file mode 100644 index 0000000000..7631af0299 --- /dev/null +++ b/mixin/thanos/lib/thanos-grafana-builder/README.md @@ -0,0 +1,6 @@ +# thanos-grafana-builder + +> Deprecated +> This will be removed soon, please do not depend on it. + +This is an `deprecated` jsonnet grafana builder library to help `thanos-mixin` to generate dashboards. diff --git a/mixin/thanos-grafana-builder/builder.libsonnet b/mixin/thanos/lib/thanos-grafana-builder/builder.libsonnet similarity index 100% rename from mixin/thanos-grafana-builder/builder.libsonnet rename to mixin/thanos/lib/thanos-grafana-builder/builder.libsonnet diff --git a/mixin/thanos-grafana-builder/grpc.libsonnet b/mixin/thanos/lib/thanos-grafana-builder/grpc.libsonnet similarity index 100% rename from mixin/thanos-grafana-builder/grpc.libsonnet rename to mixin/thanos/lib/thanos-grafana-builder/grpc.libsonnet diff --git a/mixin/thanos-grafana-builder/http.libsonnet b/mixin/thanos/lib/thanos-grafana-builder/http.libsonnet similarity index 100% rename from mixin/thanos-grafana-builder/http.libsonnet rename to mixin/thanos/lib/thanos-grafana-builder/http.libsonnet diff --git a/mixin/thanos-grafana-builder/slo.libsonnet b/mixin/thanos/lib/thanos-grafana-builder/slo.libsonnet similarity index 100% rename from mixin/thanos-grafana-builder/slo.libsonnet rename to mixin/thanos/lib/thanos-grafana-builder/slo.libsonnet From 9bb4837630aa7adb5e7f09cc2a3210237edf1b26 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 16 Dec 2019 09:00:00 -0500 Subject: [PATCH 115/257] Add tsdb store test labelnames labelvalues (#1896) * add tsdb store test labelnames labelvalues Signed-off-by: yeya24 * fix typo Signed-off-by: yeya24 --- cmd/thanos/sidecar.go | 2 +- pkg/promclient/promclient.go | 4 +- pkg/promclient/promclient_e2e_test.go | 8 +- pkg/store/tsdb_test.go | 113 ++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 7 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 859f58ac6d..282a592ee8 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -279,7 +279,7 @@ func runSidecar( } }() - if err := promclient.IsWALDirAccesible(dataDir); err != nil { + if err := promclient.IsWALDirAccessible(dataDir); err != nil { level.Error(logger).Log("err", err) } diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 582c8a9721..8474a161b5 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -37,9 +37,9 @@ import ( var ErrFlagEndpointNotFound = errors.New("no flag endpoint found") -// IsWALDirAccesible returns no error if WAL dir can be found. This helps to tell +// IsWALDirAccessible returns no error if WAL dir can be found. This helps to tell // if we have access to Prometheus TSDB directory. -func IsWALDirAccesible(dir string) error { +func IsWALDirAccessible(dir string) error { const errMsg = "WAL dir is not accessible. Is this dir a TSDB directory? If yes it is shared with TSDB?" f, err := os.Stat(filepath.Join(dir, "wal")) diff --git a/pkg/promclient/promclient_e2e_test.go b/pkg/promclient/promclient_e2e_test.go index bee767e963..adb6bbd1b7 100644 --- a/pkg/promclient/promclient_e2e_test.go +++ b/pkg/promclient/promclient_e2e_test.go @@ -19,16 +19,16 @@ import ( "github.com/thanos-io/thanos/pkg/testutil" ) -func TestIsWALFileAccesible_e2e(t *testing.T) { +func TestIsWALFileAccessible_e2e(t *testing.T) { testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) { testutil.Ok(t, p.Start()) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() - testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { return IsWALDirAccesible(p.Dir()) })) + testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { return IsWALDirAccessible(p.Dir()) })) - testutil.NotOk(t, IsWALDirAccesible(path.Join(p.Dir(), "/non-existing"))) - testutil.NotOk(t, IsWALDirAccesible(path.Join(p.Dir(), "/../"))) + testutil.NotOk(t, IsWALDirAccessible(path.Join(p.Dir(), "/non-existing"))) + testutil.NotOk(t, IsWALDirAccessible(path.Join(p.Dir(), "/../"))) }) } diff --git a/pkg/store/tsdb_test.go b/pkg/store/tsdb_test.go index 6819f74c2f..a0e8f29ddf 100644 --- a/pkg/store/tsdb_test.go +++ b/pkg/store/tsdb_test.go @@ -34,6 +34,119 @@ func TestTSDBStore_Info(t *testing.T) { testutil.Equals(t, int64(math.MaxInt64), resp.MaxTime) } +func TestTSDBStore_LabelNames(t *testing.T) { + var err error + defer leaktest.CheckTimeout(t, 10*time.Second)() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + db, err := testutil.NewTSDB() + defer func() { testutil.Ok(t, db.Close()) }() + testutil.Ok(t, err) + + appender := db.Appender() + addLabels := func(lbs []string) { + if len(lbs) > 0 { + _, err = appender.Add(labels.FromStrings(lbs...), math.MaxInt64, 1) + testutil.Ok(t, err) + } + } + + tsdbStore := NewTSDBStore(nil, nil, db, component.Rule, labels.FromStrings("region", "eu-west")) + + for _, tc := range []struct { + title string + labels []string + expectedNames []string + }{ + { + title: "no label in tsdb", + labels: []string{}, + expectedNames: []string{}, + }, + { + title: "add one label", + labels: []string{"foo", "foo"}, + expectedNames: []string{"foo"}, + }, + { + title: "add another label", + labels: []string{"bar", "bar"}, + // we will get two labels here + expectedNames: []string{"bar", "foo"}, + }, + } { + if ok := t.Run(tc.title, func(t *testing.T) { + addLabels(tc.labels) + resp, err := tsdbStore.LabelNames(ctx, &storepb.LabelNamesRequest{}) + testutil.Ok(t, err) + testutil.Equals(t, tc.expectedNames, resp.Names) + testutil.Equals(t, 0, len(resp.Warnings)) + }); !ok { + return + } + } +} + +func TestTSDBStore_LabelValues(t *testing.T) { + var err error + defer leaktest.CheckTimeout(t, 10*time.Second)() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + db, err := testutil.NewTSDB() + defer func() { testutil.Ok(t, db.Close()) }() + testutil.Ok(t, err) + + appender := db.Appender() + addLabels := func(lbs []string) { + if len(lbs) > 0 { + _, err = appender.Add(labels.FromStrings(lbs...), math.MaxInt64, 1) + testutil.Ok(t, err) + } + } + + tsdbStore := NewTSDBStore(nil, nil, db, component.Rule, labels.FromStrings("region", "eu-west")) + + for _, tc := range []struct { + title string + addedLabels []string + queryLabel string + expectedValues []string + }{ + { + title: "no label in tsdb", + addedLabels: []string{}, + queryLabel: "foo", + expectedValues: []string{}, + }, + { + title: "add one label value", + addedLabels: []string{"foo", "test"}, + queryLabel: "foo", + expectedValues: []string{"test"}, + }, + { + title: "add another label value", + addedLabels: []string{"foo", "test1"}, + queryLabel: "foo", + expectedValues: []string{"test", "test1"}, + }, + } { + if ok := t.Run(tc.title, func(t *testing.T) { + addLabels(tc.addedLabels) + resp, err := tsdbStore.LabelValues(ctx, &storepb.LabelValuesRequest{Label: tc.queryLabel}) + testutil.Ok(t, err) + testutil.Equals(t, tc.expectedValues, resp.Values) + testutil.Equals(t, 0, len(resp.Warnings)) + }); !ok { + return + } + } +} + // Regression test for https://github.com/thanos-io/thanos/issues/1038. func TestTSDBStore_Series_SplitSamplesIntoChunksWithMaxSizeOfUint16_e2e(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() From 6ba1fce673f3ab8b09c811c4218069d6e0fb4053 Mon Sep 17 00:00:00 2001 From: johncming Date: Tue, 17 Dec 2019 03:10:34 +0800 Subject: [PATCH 116/257] pkg/store: Optimize the initialization of sync.Pool. (#1892) Signed-off-by: johncming --- pkg/store/prometheus.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 42fa3f02e3..442dc7b770 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -57,6 +57,10 @@ type PrometheusStore struct { remoteReadAcceptableResponses []prompb.ReadRequest_ResponseType } +const ( + initialBufSize = 32 * 1024 // 32KB seems like a good minimum starting size. +) + // NewPrometheusStore returns a new PrometheusStore that uses the given HTTP client // to talk to Prometheus. // It attaches the provided external labels to all results. @@ -84,6 +88,10 @@ func NewPrometheusStore( externalLabels: externalLabels, timestamps: timestamps, remoteReadAcceptableResponses: []prompb.ReadRequest_ResponseType{prompb.ReadRequest_STREAMED_XOR_CHUNKS, prompb.ReadRequest_SAMPLES}, + buffers: sync.Pool{New: func() interface{} { + b := make([]byte, 0, initialBufSize) + return &b + }}, } return p, nil } @@ -121,10 +129,6 @@ func (p *PrometheusStore) Info(ctx context.Context, r *storepb.InfoRequest) (*st func (p *PrometheusStore) getBuffer() *[]byte { b := p.buffers.Get() - if b == nil { - buf := make([]byte, 0, 32*1024) // 32KB seems like a good minimum starting size. - return &buf - } return b.(*[]byte) } From f04953cc8ddd21fe8036e5831953b0773e2fd598 Mon Sep 17 00:00:00 2001 From: johncming Date: Tue, 17 Dec 2019 21:09:38 +0800 Subject: [PATCH 117/257] bugfix: adjust the timeout. (#1897) Signed-off-by: johncming --- pkg/query/storeset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/query/storeset.go b/pkg/query/storeset.go index c7afef6497..5f478c5d42 100644 --- a/pkg/query/storeset.go +++ b/pkg/query/storeset.go @@ -193,7 +193,7 @@ func NewStoreSet( storeSpecs: storeSpecs, dialOpts: dialOpts, storesMetric: storesMetric, - gRPCInfoCallTimeout: 10 * time.Second, + gRPCInfoCallTimeout: 5 * time.Second, stores: make(map[string]*storeRef), storeStatuses: make(map[string]*StoreStatus), unhealthyStoreTimeout: unhealthyStoreTimeout, From 314d3f714a03a840eecdf3ac27c2413df0224726 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Tue, 17 Dec 2019 09:07:12 -0800 Subject: [PATCH 118/257] Don't require Prometheus min/max block durations to be the same if the (#1902) user passes a flag. Signed-off-by: Callum Styan --- cmd/thanos/sidecar.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 282a592ee8..f6961e9766 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -60,6 +60,8 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { uploadCompacted := cmd.Flag("shipper.upload-compacted", "If true sidecar will try to upload compacted blocks as well. Useful for migration purposes. Works only if compaction is disabled on Prometheus. Do it once and then disable the flag when done.").Default("false").Bool() + ignoreBlockSize := cmd.Flag("shipper.ignore-unequal-block-size", "If true sidecar will not require prometheus min and max block size flags to be set to the same value. Only use this if you want to keep long retention and compaction enabled on your Prometheus instance, as in the worst case it can result in ~2h data loss for your Thanos bucket storage.").Default("false").Hidden().Bool() + minTime := thanosmodel.TimeOrDuration(cmd.Flag("min-time", "Start of time range limit to serve. Thanos sidecar will serve only metrics, which happened later than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y."). Default("0000-01-01T00:00:00Z")) @@ -90,6 +92,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { objStoreConfig, rl, *uploadCompacted, + *ignoreBlockSize, component.Sidecar, *minTime, ) @@ -114,6 +117,7 @@ func runSidecar( objStoreConfig *extflag.PathOrContent, reloader *reloader.Reloader, uploadCompacted bool, + ignoreBlockSize bool, comp component.Component, limitMinTime thanosmodel.TimeOrDurationValue, ) error { @@ -174,7 +178,7 @@ func runSidecar( // Only check Prometheus's flags when upload is enabled. if uploads { // Check prometheus's flags to ensure sane sidecar flags. - if err := validatePrometheus(ctx, logger, m); err != nil { + if err := validatePrometheus(ctx, logger, ignoreBlockSize, m); err != nil { return errors.Wrap(err, "validate Prometheus flags") } } @@ -328,7 +332,7 @@ func runSidecar( return nil } -func validatePrometheus(ctx context.Context, logger log.Logger, m *promMetadata) error { +func validatePrometheus(ctx context.Context, logger log.Logger, ignoreBlockSize bool, m *promMetadata) error { var ( flagErr error flags promclient.Flags @@ -351,10 +355,12 @@ func validatePrometheus(ctx context.Context, logger log.Logger, m *promMetadata) // Check if compaction is disabled. if flags.TSDBMinTime != flags.TSDBMaxTime { - return errors.Errorf("found that TSDB Max time is %s and Min time is %s. "+ - "Compaction needs to be disabled (storage.tsdb.min-block-duration = storage.tsdb.max-block-duration)", flags.TSDBMaxTime, flags.TSDBMinTime) + if !ignoreBlockSize { + return errors.Errorf("found that TSDB Max time is %s and Min time is %s. "+ + "Compaction needs to be disabled (storage.tsdb.min-block-duration = storage.tsdb.max-block-duration)", flags.TSDBMaxTime, flags.TSDBMinTime) + } + level.Warn(logger).Log("msg", "flag to ignore Prometheus min/max block duration flags differing is being used. If the upload of a 2h block fails and a Prometheus compaction happens that block may be missing from your Thanos bucket storage.") } - // Check if block time is 2h. if flags.TSDBMinTime != model.Duration(2*time.Hour) { level.Warn(logger).Log("msg", "found that TSDB block time is not 2h. Only 2h block time is recommended.", "block-time", flags.TSDBMinTime) From a105f963ddf613a5095ac24b68583439a6b9950d Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Wed, 18 Dec 2019 17:21:27 +0800 Subject: [PATCH 119/257] fix error in MAC OS (#1903) Signed-off-by: Xiang Dai <764524258@qq.com> --- scripts/genflagdocs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/genflagdocs.sh b/scripts/genflagdocs.sh index 549cc0a0b3..278a7a4083 100755 --- a/scripts/genflagdocs.sh +++ b/scripts/genflagdocs.sh @@ -50,7 +50,7 @@ for x in "${checkCommands[@]}"; do done # remove white noise -sed -i 's/[ \t]*$//' docs/components/flags/*.txt +sed -i -e 's/[ \t]*$//' docs/components/flags/*.txt go run scripts/cfggen/main.go --output-dir=docs/flags From 880ce3ca9a2b3e83f61391d90438bb82b140eb19 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Wed, 18 Dec 2019 10:47:18 +0100 Subject: [PATCH 120/257] scripts/build-check-comments.sh: fix exit code (#1907) Signed-off-by: Simon Pasquier --- pkg/store/tsdb_test.go | 2 +- scripts/build-check-comments.sh | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/store/tsdb_test.go b/pkg/store/tsdb_test.go index a0e8f29ddf..3b1483f194 100644 --- a/pkg/store/tsdb_test.go +++ b/pkg/store/tsdb_test.go @@ -73,7 +73,7 @@ func TestTSDBStore_LabelNames(t *testing.T) { { title: "add another label", labels: []string{"bar", "bar"}, - // we will get two labels here + // We will get two labels here. expectedNames: []string{"bar", "foo"}, }, } { diff --git a/scripts/build-check-comments.sh b/scripts/build-check-comments.sh index 8ca6cfc2f3..9648d44de5 100755 --- a/scripts/build-check-comments.sh +++ b/scripts/build-check-comments.sh @@ -94,12 +94,15 @@ function check_comments { # option is used and a selected line is found. if test "0" == "${res}" # found something then - printf "\n\n\n Error: Found comments without trailing period. Comments has to be full sentences.\n\n\n." + printf "\n\n\n Error: Found comments without trailing period. Comments has to be full sentences.\n\n\n" + exit 1 elif test "1" == "${res}" # nothing found, all clear then printf "\n\n\n All comment formatting is good, Spartan.\n\n\n" + exit 0 else # grep error printf "\n\n\n Hmmm something didn't work, issues with grep?.\n\n\n" + exit 2 fi } From db8f3470e06d654fa6c6780a823a1e01f835b9c8 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Wed, 18 Dec 2019 15:19:52 +0100 Subject: [PATCH 121/257] Makefile: fix lint and check-docs targets (#1908) Signed-off-by: Simon Pasquier --- Makefile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index a423647b3e..729a50724a 100644 --- a/Makefile +++ b/Makefile @@ -181,10 +181,7 @@ check-docs: $(EMBEDMD) $(LICHE) build @$(LICHE) --recursive docs --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root . @$(LICHE) --exclude "(cloud.tencent.com|goreportcard.com|alibabacloud.com)" --document-root . *.md @find -type f -name "*.md" | xargs scripts/cleanup-white-noise.sh - @if [[ ! git diff-files --quiet --ignore-submodules -- ]]; then \ - echo >&2 "please clean up white noise in all docs"; \ - exit 1; \ - fi + $(call require_clean_work_tree,"check documentation") # checks Go code comments if they have trailing period (excludes protobuffers and vendor files). # Comments with more than 3 spaces at beginning are omitted from the check, example: '// - foo'. @@ -304,10 +301,7 @@ lint: check-git $(GOLANGCILINT) $(MISSPELL) @find . -type f | grep -v vendor/ | grep -vE '\./\..*' | xargs $(MISSPELL) -error @echo ">> detecting white noise" @find . -type f \( -name "*.md" -o -name "*.go" \) | xargs scripts/cleanup-white-noise.sh - @if [[ ! git diff-files --quiet --ignore-submodules -- ]]; then \ - echo >&2 "please clean up white noise in all docs or Go files"; \ - exit 1; \ - fi + $(call require_clean_work_tree,"lint") .PHONY: web-serve web-serve: web-pre-process $(HUGO) From 612f53390e8d99b62a94828c82ab176b0ae7d166 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Wed, 18 Dec 2019 22:36:42 +0800 Subject: [PATCH 122/257] Add Configuring Rules (#1826) Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/components/rule.md | 84 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/docs/components/rule.md b/docs/components/rule.md index c82d82b8e7..239804ab07 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -43,6 +43,90 @@ For Ruler the read path is distributed, since most likely Ruler is querying Than This means that **query failure** are more likely to happen, that's why clear strategy on what will happen to alert and during query unavailability is the key. + +## Configuring Rules + + +Rule files use YAML, the syntax of a rule file is: + +``` +groups: + [ - ] +``` + +A simple example rules file would be: + +``` +groups: + - name: example + rules: + - record: job:http_inprogress_requests:sum + expr: sum(http_inprogress_requests) by (job) +``` + + + +``` +# The name of the group. Must be unique within a file. +name: + +# How often rules in the group are evaluated. +[ interval: | default = global.evaluation_interval ] + +rules: + [ - ... ] +``` + +Thanos supports two types of rules which may be configured and then evaluated at regular intervals: recording rules and alerting rules. + +### Recording Rules + +Recording rules allow you to precompute frequently needed or computationally expensive expressions and save their result as a new set of time series. Querying the precomputed result will then often be much faster than executing the original expression every time it is needed. This is especially useful for dashboards, which need to query the same expression repeatedly every time they refresh. + +Recording and alerting rules exist in a rule group. Rules within a group are run sequentially at a regular interval. + +The syntax for recording rules is: + +``` +# The name of the time series to output to. Must be a valid metric name. +record: + +# The PromQL expression to evaluate. Every evaluation cycle this is +# evaluated at the current time, and the result recorded as a new set of +# time series with the metric name as given by 'record'. +expr: + +# Labels to add or overwrite before storing the result. +labels: + [ : ] +``` + +### Alerting Rules + +The syntax for alerting rules is: + +``` +# The name of the alert. Must be a valid metric name. +alert: + +# The PromQL expression to evaluate. Every evaluation cycle this is +# evaluated at the current time, and all resultant time series become +# pending/firing alerts. +expr: + +# Alerts are considered firing once they have been returned for this long. +# Alerts which have not yet fired for long enough are considered pending. +[ for: | default = 0s ] + +# Labels to add or overwrite for each alert. +labels: + [ : ] + +# Annotations to add to each alert. +annotations: + [ : ] +``` + ## Partial Response See [this](query.md#partial-response) on initial info. From 56abeab3777d8cb8ff09bf45bfaacce3cf782ffc Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Wed, 18 Dec 2019 17:32:06 +0100 Subject: [PATCH 123/257] *: Added support for authentication and TLS for Alertmanager (#1838) * *: support authentication and TLS for Alertmanager This change adds support for authentication with basic auth, client certificates and bearer tokens. It also enables to configure TLS settings for the Alertmanager endpoints. Most of the work leverages the existing Prometheus configuration format and code. In particular TLS certificate files are automatically reloaded whenever they change. Signed-off-by: Simon Pasquier * Fail hard when --alertmanagers.url and --alertmanagers.config flags are both defined Signed-off-by: Simon Pasquier * Update CHANGELOG.md Signed-off-by: Simon Pasquier * Move tests from cmd/thanos to pkg/alert Signed-off-by: Simon Pasquier * Add end-to-end for Alertmanager file SD Signed-off-by: Simon Pasquier * test/e2e: add test with different alerting HTTP clients Signed-off-by: Simon Pasquier * Fix panic in pkg/alert/client_test.go Signed-off-by: Simon Pasquier * Address Bartek's comments Signed-off-by: Simon Pasquier * Re-use dns.Provider for resolving Alertmanager addresses Signed-off-by: Simon Pasquier * update documentation Signed-off-by: Simon Pasquier --- CHANGELOG.md | 2 + cmd/thanos/rule.go | 192 +++++++++---------- cmd/thanos/rule_test.go | 98 ---------- docs/components/rule.md | 55 +++++- docs/service-discovery.md | 6 +- pkg/alert/alert.go | 109 ++++------- pkg/alert/alert_test.go | 125 ++++++------- pkg/alert/client.go | 343 ++++++++++++++++++++++++++++++++++ pkg/alert/client_test.go | 88 +++++++++ pkg/discovery/dns/provider.go | 12 ++ scripts/cfggen/main.go | 24 ++- test/e2e/rule_test.go | 332 +++++++++++++++++++++++++++++++- test/e2e/spinup_test.go | 6 +- 13 files changed, 1020 insertions(+), 372 deletions(-) create mode 100644 pkg/alert/client.go create mode 100644 pkg/alert/client_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c3249140ed..b4634cb1dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Added - [#1852](https://github.com/thanos-io/thanos/pull/1852) Add support for `AWS_CONTAINER_CREDENTIALS_FULL_URI` by upgrading to minio-go v6.0.44 - [#1854](https://github.com/thanos-io/thanos/pull/1854) Update Rule UI to support alerts count displaying and filtering. +- [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add TLS and authentication support for Alertmanager with the `--alertmanagers.config` and `--alertmanagers.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. +- [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add a new `--alertmanagers.sd-dns-interval` CLI option to specify the interval between DNS resolutions of Alertmanager hosts. ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 115be52ff8..9e3af8c8d4 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/rand" - "net" "net/http" "net/url" "os" @@ -13,7 +12,6 @@ import ( "path/filepath" "strconv" "strings" - "sync" "syscall" "time" @@ -83,8 +81,10 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { alertmgrs := cmd.Flag("alertmanagers.url", "Alertmanager replica URLs to push firing alerts. Ruler claims success if push to at least one alertmanager from discovered succeeds. The scheme should not be empty e.g `http` might be used. The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Alertmanager IPs through respective DNS lookups. The port defaults to 9093 or the SRV record's value. The URL path is used as a prefix for the regular Alertmanager API path."). Strings() - - alertmgrsTimeout := cmd.Flag("alertmanagers.send-timeout", "Timeout for sending alerts to alertmanager").Default("10s").Duration() + alertmgrsTimeout := cmd.Flag("alertmanagers.send-timeout", "Timeout for sending alerts to Alertmanager").Default("10s").Duration() + alertmgrsConfig := extflag.RegisterPathOrContent(cmd, "alertmanagers.config", "YAML file that contains alerting configuration. See format details: https://thanos.io/components/rule.md/#configuration. If defined, it takes precedence over the '--alertmanagers.url' and '--alertmanagers.send-timeout' flags.", false) + alertmgrsDNSSDInterval := modelDuration(cmd.Flag("alertmanagers.sd-dns-interval", "Interval between DNS resolutions of Alertmanager hosts."). + Default("30s")) alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() @@ -157,6 +157,8 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { lset, *alertmgrs, *alertmgrsTimeout, + alertmgrsConfig, + time.Duration(*alertmgrsDNSSDInterval), *grpcBindAddr, time.Duration(*grpcGracePeriod), *grpcCert, @@ -194,6 +196,8 @@ func runRule( lset labels.Labels, alertmgrURLs []string, alertmgrsTimeout time.Duration, + alertmgrsConfig *extflag.PathOrContent, + alertmgrsDNSSDInterval time.Duration, grpcBindAddr string, grpcGracePeriod time.Duration, grpcCert string, @@ -286,11 +290,56 @@ func runRule( dns.ResolverType(dnsSDResolver), ) + // Build the Alertmanager clients. + alertmgrsConfigYAML, err := alertmgrsConfig.Content() + if err != nil { + return err + } + var ( + alertingCfg alert.AlertingConfig + alertmgrs []*alert.Alertmanager + ) + if len(alertmgrsConfigYAML) > 0 { + if len(alertmgrURLs) != 0 { + return errors.New("--alertmanagers.url and --alertmanagers.config* flags cannot be defined at the same time") + } + alertingCfg, err = alert.LoadAlertingConfig(alertmgrsConfigYAML) + if err != nil { + return err + } + } else { + // Build the Alertmanager configuration from the legacy flags. + for _, addr := range alertmgrURLs { + cfg, err := alert.BuildAlertmanagerConfig(logger, addr, alertmgrsTimeout) + if err != nil { + return err + } + alertingCfg.Alertmanagers = append(alertingCfg.Alertmanagers, cfg) + } + } + + if len(alertingCfg.Alertmanagers) == 0 { + level.Warn(logger).Log("msg", "no alertmanager configured") + } + + amProvider := dns.NewProvider( + logger, + extprom.WrapRegistererWithPrefix("thanos_ruler_alertmanagers_", reg), + dns.ResolverType(dnsSDResolver), + ) + for _, cfg := range alertingCfg.Alertmanagers { + // Each Alertmanager client has a different list of targets thus each needs its own DNS provider. + am, err := alert.NewAlertmanager(logger, cfg, amProvider.Clone()) + if err != nil { + return err + } + alertmgrs = append(alertmgrs, am) + } + // Run rule evaluation and alert notifications. var ( - alertmgrs = newAlertmanagerSet(logger, alertmgrURLs, dns.ResolverType(dnsSDResolver)) - alertQ = alert.NewQueue(logger, reg, 10000, 100, labelsTSDBToProm(lset), alertExcludeLabels) - ruleMgr = thanosrule.NewManager(dataDir) + alertQ = alert.NewQueue(logger, reg, 10000, 100, labelsTSDBToProm(lset), alertExcludeLabels) + ruleMgr = thanosrule.NewManager(dataDir) ) { notify := func(ctx context.Context, expr string, alerts ...*rules.Alert) { @@ -351,9 +400,35 @@ func runRule( }) } } + // Discover and resolve Alertmanager addresses. + { + for i := range alertmgrs { + am := alertmgrs[i] + ctx, cancel := context.WithCancel(context.Background()) + g.Add(func() error { + am.Discover(ctx) + return nil + }, func(error) { + cancel() + }) + + g.Add(func() error { + return runutil.Repeat(alertmgrsDNSSDInterval, ctx.Done(), func() error { + am.Resolve(ctx) + return nil + }) + }, func(error) { + cancel() + }) + } + } + // Run the alert sender. { - // TODO(bwplotka): https://github.com/thanos-io/thanos/issues/660. - sdr := alert.NewSender(logger, reg, alertmgrs.get, nil, alertmgrsTimeout) + clients := make([]alert.AlertmanagerClient, len(alertmgrs)) + for i := range alertmgrs { + clients[i] = alertmgrs[i] + } + sdr := alert.NewSender(logger, reg, clients) ctx, cancel := context.WithCancel(context.Background()) g.Add(func() error { @@ -370,21 +445,6 @@ func runRule( cancel() }) } - { - ctx, cancel := context.WithCancel(context.Background()) - - g.Add(func() error { - return runutil.Repeat(30*time.Second, ctx.Done(), func() error { - if err := alertmgrs.update(ctx); err != nil { - level.Error(logger).Log("msg", "refreshing alertmanagers failed", "err", err) - alertMngrAddrResolutionErrors.Inc() - } - return nil - }) - }, func(error) { - cancel() - }) - } // Run File Service Discovery and update the query addresses when the files are modified. if fileSD != nil { var fileSDUpdates chan []*targetgroup.Group @@ -615,90 +675,6 @@ func runRule( return nil } -type alertmanagerSet struct { - resolver dns.Resolver - addrs []string - mtx sync.Mutex - current []*url.URL -} - -func newAlertmanagerSet(logger log.Logger, addrs []string, dnsSDResolver dns.ResolverType) *alertmanagerSet { - return &alertmanagerSet{ - resolver: dns.NewResolver(dnsSDResolver.ToResolver(logger)), - addrs: addrs, - } -} - -func (s *alertmanagerSet) get() []*url.URL { - s.mtx.Lock() - defer s.mtx.Unlock() - return s.current -} - -const defaultAlertmanagerPort = 9093 - -func parseAlertmanagerAddress(addr string) (qType dns.QType, parsedUrl *url.URL, err error) { - qType = "" - parsedUrl, err = url.Parse(addr) - if err != nil { - return qType, nil, err - } - // The Scheme might contain DNS resolver type separated by + so we split it a part. - if schemeParts := strings.Split(parsedUrl.Scheme, "+"); len(schemeParts) > 1 { - parsedUrl.Scheme = schemeParts[len(schemeParts)-1] - qType = dns.QType(strings.Join(schemeParts[:len(schemeParts)-1], "+")) - } - return qType, parsedUrl, err -} - -func (s *alertmanagerSet) update(ctx context.Context) error { - var result []*url.URL - for _, addr := range s.addrs { - var ( - qtype dns.QType - resolvedDomain []string - ) - - qtype, u, err := parseAlertmanagerAddress(addr) - if err != nil { - return errors.Wrapf(err, "parse URL %q", addr) - } - - // Get only the host and resolve it if needed. - host := u.Host - if qtype != "" { - if qtype == dns.A { - _, _, err = net.SplitHostPort(host) - if err != nil { - // The host could be missing a port. Append the defaultAlertmanagerPort. - host = host + ":" + strconv.Itoa(defaultAlertmanagerPort) - } - } - resolvedDomain, err = s.resolver.Resolve(ctx, host, qtype) - if err != nil { - return errors.Wrap(err, "alertmanager resolve") - } - } else { - resolvedDomain = []string{host} - } - - for _, resolved := range resolvedDomain { - result = append(result, &url.URL{ - Scheme: u.Scheme, - Host: resolved, - Path: u.Path, - User: u.User, - }) - } - } - - s.mtx.Lock() - s.current = result - s.mtx.Unlock() - - return nil -} - func parseFlagLabels(s []string) (labels.Labels, error) { var lset labels.Labels for _, l := range s { diff --git a/cmd/thanos/rule_test.go b/cmd/thanos/rule_test.go index f4d801d747..2abd38cf6a 100644 --- a/cmd/thanos/rule_test.go +++ b/cmd/thanos/rule_test.go @@ -1,12 +1,8 @@ package main import ( - "context" - "net/url" "testing" - "github.com/pkg/errors" - "github.com/thanos-io/thanos/pkg/discovery/dns" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -49,97 +45,3 @@ func Test_parseFlagLabels(t *testing.T) { testutil.Equals(t, err != nil, td.expectErr) } } - -func TestRule_AlertmanagerResolveWithoutPort(t *testing.T) { - mockResolver := mockResolver{ - resultIPs: map[string][]string{ - "alertmanager.com:9093": {"1.1.1.1:9300"}, - }, - } - am := alertmanagerSet{resolver: mockResolver, addrs: []string{"dns+http://alertmanager.com"}} - - ctx := context.TODO() - err := am.update(ctx) - testutil.Ok(t, err) - - expected := []*url.URL{ - { - Scheme: "http", - Host: "1.1.1.1:9300", - }, - } - gotURLs := am.get() - testutil.Equals(t, expected, gotURLs) -} - -func TestRule_AlertmanagerResolveWithPort(t *testing.T) { - mockResolver := mockResolver{ - resultIPs: map[string][]string{ - "alertmanager.com:19093": {"1.1.1.1:9300"}, - }, - } - am := alertmanagerSet{resolver: mockResolver, addrs: []string{"dns+http://alertmanager.com:19093"}} - - ctx := context.TODO() - err := am.update(ctx) - testutil.Ok(t, err) - - expected := []*url.URL{ - { - Scheme: "http", - Host: "1.1.1.1:9300", - }, - } - gotURLs := am.get() - testutil.Equals(t, expected, gotURLs) -} - -type mockResolver struct { - resultIPs map[string][]string - err error -} - -func (m mockResolver) Resolve(ctx context.Context, name string, qtype dns.QType) ([]string, error) { - if m.err != nil { - return nil, m.err - } - if res, ok := m.resultIPs[name]; ok { - return res, nil - } - return nil, errors.Errorf("mockResolver not found response for name: %s", name) -} - -func Test_ParseAlertmanagerAddress(t *testing.T) { - var tData = []struct { - address string - expectQueryType dns.QType - expectUrl *url.URL - expectError error - }{ - { - address: "http://user:pass+word@foo.bar:3289", - expectQueryType: dns.QType(""), - expectUrl: &url.URL{Host: "foo.bar:3289", Scheme: "http", User: url.UserPassword("user", "pass+word")}, - expectError: nil, - }, - { - address: "dnssrvnoa+http://user:pass+word@foo.bar:3289", - expectQueryType: dns.QType("dnssrvnoa"), - expectUrl: &url.URL{Host: "foo.bar:3289", Scheme: "http", User: url.UserPassword("user", "pass+word")}, - expectError: nil, - }, - { - address: "foo+bar+http://foo.bar:3289", - expectQueryType: dns.QType("foo+bar"), - expectUrl: &url.URL{Host: "foo.bar:3289", Scheme: "http"}, - expectError: nil, - }, - } - - for _, d := range tData { - q, u, e := parseAlertmanagerAddress(d.address) - testutil.Equals(t, d.expectError, e) - testutil.Equals(t, d.expectUrl, u) - testutil.Equals(t, d.expectQueryType, q) - } -} diff --git a/docs/components/rule.md b/docs/components/rule.md index 239804ab07..ea6670c63e 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -293,7 +293,26 @@ Flags: record's value. The URL path is used as a prefix for the regular Alertmanager API path. --alertmanagers.send-timeout=10s - Timeout for sending alerts to alertmanager + Timeout for sending alerts to Alertmanager + --alertmanagers.config-file= + Path to YAML file that contains alerting + configuration. See format details: + https://thanos.io/components/rule.md/#configuration. + If defined, it takes precedence over the + '--alertmanagers.url' and + '--alertmanagers.send-timeout' flags. + --alertmanagers.config= + Alternative to 'alertmanagers.config-file' flag + (lower priority). Content of YAML file that + contains alerting configuration. See format + details: + https://thanos.io/components/rule.md/#configuration. + If defined, it takes precedence over the + '--alertmanagers.url' and + '--alertmanagers.send-timeout' flags. + --alertmanagers.sd-dns-interval=30s + Interval between DNS resolutions of + Alertmanager hosts. --alert.query-url=ALERT.QUERY-URL The external Thanos Query URL that would be set in all alerts 'Source' field @@ -350,3 +369,37 @@ Flags: Interval between DNS resolutions. ``` + +## Configuration + +### Alertmanager + +The `--alertmanagers.config` and `--alertmanagers.config-file` flags allow specifying multiple Alertmanagers. Those entries are treated as a single HA group. This means that alert send failure is claimed only if the Ruler fails to send to all instances. + +The configuration format is the following: + +[embedmd]:# (../flags/config_rule_alerting.txt yaml) +```yaml +alertmanagers: +- http_config: + basic_auth: + username: "" + password: "" + password_file: "" + bearer_token: "" + bearer_token_file: "" + proxy_url: "" + tls_config: + ca_file: "" + cert_file: "" + key_file: "" + server_name: "" + insecure_skip_verify: false + static_configs: [] + file_sd_configs: + - files: [] + refresh_interval: 0s + scheme: http + path_prefix: "" + timeout: 10s +``` diff --git a/docs/service-discovery.md b/docs/service-discovery.md index cdb44e8493..34cf0914cd 100644 --- a/docs/service-discovery.md +++ b/docs/service-discovery.md @@ -13,7 +13,7 @@ SD is currently used in the following places within Thanos: * `Thanos Query` needs to know about [StoreAPI](https://github.com/thanos-io/thanos/blob/d3fb337da94d11c78151504b1fccb1d7e036f394/pkg/store/storepb/rpc.proto#L14) servers in order to query metrics from them. * `Thanos Rule` needs to know about `QueryAPI` servers in order to evaluate recording and alerting rules. -* `Thanos Rule` needs to know about `Alertmanagers` HA replicas in order to send alerts; only static option with DNS discovery. +* `Thanos Rule` needs to know about `Alertmanagers` HA replicas in order to send alerts. There are currently several ways to configure SD, described below in more detail: @@ -33,7 +33,7 @@ The repeatable flag `--store=` can be used to specify a `StoreAPI` that ` The repeatable flag `--query=` can be used to specify a `QueryAPI` that `Thanos Rule` should use. -The repeatable flag `--alertmanager.url=` can be used to specify a `Alertmanager API` that `Thanos Rule` should use. +`Thanos Rule` also supports the configuration of Alertmanager endpoints using YAML with the `--alertmanagers.config=` and `--alertmanagers.config-file=` flags in the `StaticAddress` section. ## File Service Discovery @@ -77,6 +77,8 @@ Again, the `` can be a glob pattern. The flag `--query.sd-interval=<5m>` can be used to change the fallback re-read interval. +`Thanos Rule` also supports the configuration of Alertmanager endpoints using YAML with the `--alertmanagers.config=` and `--alertmanagers.config-file=` flags in the `FileSDfiles` section.. + ## DNS Service Discovery DNS Service Discovery is another mechanism for finding components that can be used in conjunction with Static Flags or File SD. diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go index 6af16674e8..2a9e2e8ab8 100644 --- a/pkg/alert/alert.go +++ b/pkg/alert/alert.go @@ -6,30 +6,19 @@ import ( "context" "encoding/json" "fmt" - "net/http" + "io" "net/url" - "path" "sync" "sync/atomic" "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/version" - "github.com/prometheus/prometheus/pkg/labels" - - "github.com/thanos-io/thanos/pkg/runutil" -) -const ( - alertPushEndpoint = "/api/v1/alerts" - contentTypeJSON = "application/json" + "github.com/prometheus/prometheus/pkg/labels" ) -var userAgent = fmt.Sprintf("Thanos/%s", version.Version) - // Alert is a generic representation of an alert in the Prometheus eco-system. type Alert struct { // Label value pairs for purpose of aggregation, matching, and disposition @@ -251,12 +240,15 @@ func (q *Queue) Push(alerts []*Alert) { } } +type AlertmanagerClient interface { + Endpoints() []*url.URL + Do(context.Context, *url.URL, io.Reader) error +} + // Sender sends notifications to a dynamic set of alertmanagers. type Sender struct { logger log.Logger - alertmanagers func() []*url.URL - doReq func(req *http.Request) (*http.Response, error) - timeout time.Duration + alertmanagers []AlertmanagerClient sent *prometheus.CounterVec errs *prometheus.CounterVec @@ -269,21 +261,14 @@ type Sender struct { func NewSender( logger log.Logger, reg prometheus.Registerer, - alertmanagers func() []*url.URL, - doReq func(req *http.Request) (*http.Response, error), - timeout time.Duration, + alertmanagers []AlertmanagerClient, ) *Sender { - if doReq == nil { - doReq = http.DefaultClient.Do - } if logger == nil { logger = log.NewNopLogger() } s := &Sender{ logger: logger, alertmanagers: alertmanagers, - doReq: doReq, - timeout: timeout, sent: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "thanos_alert_sender_alerts_sent_total", @@ -311,7 +296,7 @@ func NewSender( return s } -// Send an alert batch to all given Alertmanager URLs. +// Send an alert batch to all given Alertmanager clients. // TODO(bwplotka): https://github.com/thanos-io/thanos/issues/660. func (s *Sender) Send(ctx context.Context, alerts []*Alert) { if len(alerts) == 0 { @@ -327,33 +312,30 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { wg sync.WaitGroup numSuccess uint64 ) - amrs := s.alertmanagers() - for _, u := range amrs { - amURL := *u - sendCtx, cancel := context.WithTimeout(ctx, s.timeout) - - wg.Add(1) - go func() { - defer wg.Done() - defer cancel() - - start := time.Now() - amURL.Path = path.Join(amURL.Path, alertPushEndpoint) - - if err := s.sendOne(sendCtx, amURL.String(), b); err != nil { - level.Warn(s.logger).Log( - "msg", "sending alerts failed", - "alertmanager", amURL.Host, - "numAlerts", len(alerts), - "err", err) - s.errs.WithLabelValues(amURL.Host).Inc() - return - } - s.latency.WithLabelValues(amURL.Host).Observe(time.Since(start).Seconds()) - s.sent.WithLabelValues(amURL.Host).Add(float64(len(alerts))) - - atomic.AddUint64(&numSuccess, 1) - }() + for _, amc := range s.alertmanagers { + for _, u := range amc.Endpoints() { + wg.Add(1) + go func(amc AlertmanagerClient, u *url.URL) { + defer wg.Done() + + level.Debug(s.logger).Log("msg", "sending alerts", "alertmanager", u.Host, "numAlerts", len(alerts)) + start := time.Now() + if err := amc.Do(ctx, u, bytes.NewReader(b)); err != nil { + level.Warn(s.logger).Log( + "msg", "sending alerts failed", + "alertmanager", u.Host, + "numAlerts", len(alerts), + "err", err, + ) + s.errs.WithLabelValues(u.Host).Inc() + return + } + s.latency.WithLabelValues(u.Host).Observe(time.Since(start).Seconds()) + s.sent.WithLabelValues(u.Host).Add(float64(len(alerts))) + + atomic.AddUint64(&numSuccess, 1) + }(amc, u) + } } wg.Wait() @@ -362,26 +344,5 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { } s.dropped.Add(float64(len(alerts))) - level.Warn(s.logger).Log("msg", "failed to send alerts to all alertmanagers", "alertmanagers", amrs, "alerts", string(b)) -} - -func (s *Sender) sendOne(ctx context.Context, url string, b []byte) error { - req, err := http.NewRequest("POST", url, bytes.NewReader(b)) - if err != nil { - return err - } - req = req.WithContext(ctx) - req.Header.Set("Content-Type", contentTypeJSON) - req.Header.Set("User-Agent", userAgent) - - resp, err := s.doReq(req) - if err != nil { - return errors.Wrapf(err, "send request to %q", url) - } - defer runutil.ExhaustCloseWithLogOnErr(s.logger, resp.Body, "send one alert") - - if resp.StatusCode/100 != 2 { - return errors.Errorf("bad response status %v from %q", resp.Status, url) - } - return nil + level.Warn(s.logger).Log("msg", "failed to send alerts to all alertmanagers", "alerts", string(b)) } diff --git a/pkg/alert/alert_test.go b/pkg/alert/alert_test.go index c2fcc8f91a..7e1e851d80 100644 --- a/pkg/alert/alert_test.go +++ b/pkg/alert/alert_test.go @@ -1,14 +1,11 @@ package alert import ( - "bytes" "context" - "io/ioutil" - "net/http" + "io" "net/url" "sync" "testing" - "time" "github.com/prometheus/prometheus/pkg/labels" @@ -50,98 +47,86 @@ func assertSameHosts(t *testing.T, expected []*url.URL, found []*url.URL) { } } -func TestSender_Send_OK(t *testing.T) { - var ( - expectedHosts = []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}} - spottedHosts []*url.URL - spottedMu sync.Mutex - ) +type fakeClient struct { + urls []*url.URL + postf func(u *url.URL) error + mtx sync.Mutex + seen []*url.URL +} - okDo := func(req *http.Request) (response *http.Response, e error) { - spottedMu.Lock() - defer spottedMu.Unlock() +func (f *fakeClient) Endpoints() []*url.URL { + return f.urls +} - spottedHosts = append(spottedHosts, req.URL) +func (f *fakeClient) Do(ctx context.Context, u *url.URL, r io.Reader) error { + f.mtx.Lock() + defer f.mtx.Unlock() + f.seen = append(f.seen, u) + if f.postf == nil { + return nil + } + return f.postf(u) +} - return &http.Response{ - Body: ioutil.NopCloser(bytes.NewBuffer(nil)), - StatusCode: http.StatusOK, - }, nil +func TestSenderSendsOk(t *testing.T) { + poster := &fakeClient{ + urls: []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}}, } - s := NewSender(nil, nil, func() []*url.URL { return expectedHosts }, okDo, 10*time.Second) + s := NewSender(nil, nil, []AlertmanagerClient{poster}) s.Send(context.Background(), []*Alert{{}, {}}) - assertSameHosts(t, expectedHosts, spottedHosts) + assertSameHosts(t, poster.urls, poster.seen) - testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sent.WithLabelValues(expectedHosts[0].Host)))) - testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.errs.WithLabelValues(expectedHosts[0].Host)))) + testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sent.WithLabelValues(poster.urls[0].Host)))) + testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.errs.WithLabelValues(poster.urls[0].Host)))) - testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sent.WithLabelValues(expectedHosts[1].Host)))) - testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.errs.WithLabelValues(expectedHosts[1].Host)))) + testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sent.WithLabelValues(poster.urls[1].Host)))) + testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.errs.WithLabelValues(poster.urls[1].Host)))) testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.dropped))) } -func TestSender_Send_OneFails(t *testing.T) { - var ( - expectedHosts = []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}} - spottedHosts []*url.URL - spottedMu sync.Mutex - ) - - do := func(req *http.Request) (response *http.Response, e error) { - spottedMu.Lock() - defer spottedMu.Unlock() - - spottedHosts = append(spottedHosts, req.URL) - - if req.Host == expectedHosts[0].Host { - return nil, errors.New("no such host") - } - return &http.Response{ - Body: ioutil.NopCloser(bytes.NewBuffer(nil)), - StatusCode: http.StatusOK, - }, nil +func TestSenderSendsOneFails(t *testing.T) { + poster := &fakeClient{ + urls: []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}}, + postf: func(u *url.URL) error { + if u.Host == "am1:9090" { + return errors.New("no such host") + } + return nil + }, } - s := NewSender(nil, nil, func() []*url.URL { return expectedHosts }, do, 10*time.Second) + s := NewSender(nil, nil, []AlertmanagerClient{poster}) s.Send(context.Background(), []*Alert{{}, {}}) - assertSameHosts(t, expectedHosts, spottedHosts) + assertSameHosts(t, poster.urls, poster.seen) - testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.sent.WithLabelValues(expectedHosts[0].Host)))) - testutil.Equals(t, 1, int(promtestutil.ToFloat64(s.errs.WithLabelValues(expectedHosts[0].Host)))) + testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.sent.WithLabelValues(poster.urls[0].Host)))) + testutil.Equals(t, 1, int(promtestutil.ToFloat64(s.errs.WithLabelValues(poster.urls[0].Host)))) - testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sent.WithLabelValues(expectedHosts[1].Host)))) - testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.errs.WithLabelValues(expectedHosts[1].Host)))) + testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.sent.WithLabelValues(poster.urls[1].Host)))) + testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.errs.WithLabelValues(poster.urls[1].Host)))) testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.dropped))) } -func TestSender_Send_AllFails(t *testing.T) { - var ( - expectedHosts = []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}} - spottedHosts []*url.URL - spottedMu sync.Mutex - ) - - do := func(req *http.Request) (response *http.Response, e error) { - spottedMu.Lock() - defer spottedMu.Unlock() - - spottedHosts = append(spottedHosts, req.URL) - - return nil, errors.New("no such host") +func TestSenderSendsAllFail(t *testing.T) { + poster := &fakeClient{ + urls: []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}}, + postf: func(u *url.URL) error { + return errors.New("no such host") + }, } - s := NewSender(nil, nil, func() []*url.URL { return expectedHosts }, do, 10*time.Second) + s := NewSender(nil, nil, []AlertmanagerClient{poster}) s.Send(context.Background(), []*Alert{{}, {}}) - assertSameHosts(t, expectedHosts, spottedHosts) + assertSameHosts(t, poster.urls, poster.seen) - testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.sent.WithLabelValues(expectedHosts[0].Host)))) - testutil.Equals(t, 1, int(promtestutil.ToFloat64(s.errs.WithLabelValues(expectedHosts[0].Host)))) + testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.sent.WithLabelValues(poster.urls[0].Host)))) + testutil.Equals(t, 1, int(promtestutil.ToFloat64(s.errs.WithLabelValues(poster.urls[0].Host)))) - testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.sent.WithLabelValues(expectedHosts[1].Host)))) - testutil.Equals(t, 1, int(promtestutil.ToFloat64(s.errs.WithLabelValues(expectedHosts[1].Host)))) + testutil.Equals(t, 0, int(promtestutil.ToFloat64(s.sent.WithLabelValues(poster.urls[1].Host)))) + testutil.Equals(t, 1, int(promtestutil.ToFloat64(s.errs.WithLabelValues(poster.urls[1].Host)))) testutil.Equals(t, 2, int(promtestutil.ToFloat64(s.dropped))) } diff --git a/pkg/alert/client.go b/pkg/alert/client.go new file mode 100644 index 0000000000..f58b995a2b --- /dev/null +++ b/pkg/alert/client.go @@ -0,0 +1,343 @@ +package alert + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "sync" + "time" + + "github.com/go-kit/kit/log" + "github.com/pkg/errors" + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/version" + "github.com/prometheus/prometheus/discovery/file" + "github.com/prometheus/prometheus/discovery/targetgroup" + "gopkg.in/yaml.v2" + + "github.com/thanos-io/thanos/pkg/discovery/cache" + "github.com/thanos-io/thanos/pkg/discovery/dns" + "github.com/thanos-io/thanos/pkg/runutil" +) + +const ( + defaultAlertmanagerPort = 9093 + alertPushEndpoint = "/api/v1/alerts" + contentTypeJSON = "application/json" +) + +var userAgent = fmt.Sprintf("Thanos/%s", version.Version) + +type AlertingConfig struct { + Alertmanagers []AlertmanagerConfig `yaml:"alertmanagers"` +} + +// AlertmanagerConfig represents a client to a cluster of Alertmanager endpoints. +// TODO(simonpasquier): add support for API version (v1 or v2). +type AlertmanagerConfig struct { + // HTTP client configuration. + HTTPClientConfig HTTPClientConfig `yaml:"http_config"` + + // List of addresses with DNS prefixes. + StaticAddresses []string `yaml:"static_configs"` + // List of file configurations (our FileSD supports different DNS lookups). + FileSDConfigs []FileSDConfig `yaml:"file_sd_configs"` + + // The URL scheme to use when talking to Alertmanagers. + Scheme string `yaml:"scheme"` + + // Path prefix to add in front of the push endpoint path. + PathPrefix string `yaml:"path_prefix"` + + // The timeout used when sending alerts (default: 10s). + Timeout model.Duration `yaml:"timeout"` +} + +type HTTPClientConfig struct { + // The HTTP basic authentication credentials for the targets. + BasicAuth BasicAuth `yaml:"basic_auth"` + // The bearer token for the targets. + BearerToken string `yaml:"bearer_token"` + // The bearer token file for the targets. + BearerTokenFile string `yaml:"bearer_token_file"` + // HTTP proxy server to use to connect to the targets. + ProxyURL string `yaml:"proxy_url"` + // TLSConfig to use to connect to the targets. + TLSConfig TLSConfig `yaml:"tls_config"` +} + +type TLSConfig struct { + // The CA cert to use for the targets. + CAFile string `yaml:"ca_file"` + // The client cert file for the targets. + CertFile string `yaml:"cert_file"` + // The client key file for the targets. + KeyFile string `yaml:"key_file"` + // Used to verify the hostname for the targets. + ServerName string `yaml:"server_name"` + // Disable target certificate validation. + InsecureSkipVerify bool `yaml:"insecure_skip_verify"` +} + +type BasicAuth struct { + Username string `yaml:"username"` + Password string `yaml:"password"` + PasswordFile string `yaml:"password_file"` +} + +func (b BasicAuth) IsZero() bool { + return b.Username == "" && b.Password == "" && b.PasswordFile == "" +} + +func (c HTTPClientConfig) convert() (config_util.HTTPClientConfig, error) { + httpClientConfig := config_util.HTTPClientConfig{ + BearerToken: config_util.Secret(c.BearerToken), + BearerTokenFile: c.BearerTokenFile, + TLSConfig: config_util.TLSConfig{ + CAFile: c.TLSConfig.CAFile, + CertFile: c.TLSConfig.CertFile, + KeyFile: c.TLSConfig.KeyFile, + ServerName: c.TLSConfig.ServerName, + InsecureSkipVerify: c.TLSConfig.InsecureSkipVerify, + }, + } + if c.ProxyURL != "" { + var proxy config_util.URL + err := yaml.Unmarshal([]byte(c.ProxyURL), &proxy) + if err != nil { + return httpClientConfig, err + } + httpClientConfig.ProxyURL = proxy + } + if !c.BasicAuth.IsZero() { + httpClientConfig.BasicAuth = &config_util.BasicAuth{ + Username: c.BasicAuth.Username, + Password: config_util.Secret(c.BasicAuth.Password), + PasswordFile: c.BasicAuth.PasswordFile, + } + } + return httpClientConfig, httpClientConfig.Validate() +} + +type FileSDConfig struct { + Files []string `yaml:"files"` + RefreshInterval model.Duration `yaml:"refresh_interval"` +} + +func (c FileSDConfig) convert() (file.SDConfig, error) { + var fileSDConfig file.SDConfig + b, err := yaml.Marshal(c) + if err != nil { + return fileSDConfig, err + } + err = yaml.Unmarshal(b, &fileSDConfig) + if err != nil { + return fileSDConfig, err + } + return fileSDConfig, nil +} + +func DefaultAlertmanagerConfig() AlertmanagerConfig { + return AlertmanagerConfig{ + Scheme: "http", + Timeout: model.Duration(time.Second * 10), + StaticAddresses: []string{}, + FileSDConfigs: []FileSDConfig{}, + } +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultAlertmanagerConfig() + type plain AlertmanagerConfig + return unmarshal((*plain)(c)) +} + +type AddressProvider interface { + Resolve(context.Context, []string) + Addresses() []string +} + +// Alertmanager represents an HTTP client that can send alerts to a cluster of Alertmanager endpoints. +type Alertmanager struct { + logger log.Logger + + client *http.Client + timeout time.Duration + scheme string + prefix string + + staticAddresses []string + fileSDCache *cache.Cache + fileDiscoverers []*file.Discovery + + provider AddressProvider +} + +// NewAlertmanager returns a new Alertmanager client. +func NewAlertmanager(logger log.Logger, cfg AlertmanagerConfig, provider AddressProvider) (*Alertmanager, error) { + if logger == nil { + logger = log.NewNopLogger() + } + + httpClientConfig, err := cfg.HTTPClientConfig.convert() + if err != nil { + return nil, err + } + client, err := config_util.NewClientFromConfig(httpClientConfig, "alertmanager", false) + if err != nil { + return nil, err + } + + var discoverers []*file.Discovery + for _, sdCfg := range cfg.FileSDConfigs { + fileSDCfg, err := sdCfg.convert() + if err != nil { + return nil, err + } + discoverers = append(discoverers, file.NewDiscovery(&fileSDCfg, logger)) + } + return &Alertmanager{ + logger: logger, + client: client, + scheme: cfg.Scheme, + prefix: cfg.PathPrefix, + timeout: time.Duration(cfg.Timeout), + staticAddresses: cfg.StaticAddresses, + fileSDCache: cache.New(), + fileDiscoverers: discoverers, + provider: provider, + }, nil +} + +// LoadAlertmanagerConfigs loads a list of AlertmanagerConfig from YAML data. +func LoadAlertingConfig(confYaml []byte) (AlertingConfig, error) { + var cfg AlertingConfig + if err := yaml.UnmarshalStrict(confYaml, &cfg); err != nil { + return cfg, err + } + return cfg, nil +} + +// BuildAlertmanagerConfig initializes and returns an Alertmanager client configuration from a static address. +func BuildAlertmanagerConfig(logger log.Logger, address string, timeout time.Duration) (AlertmanagerConfig, error) { + parsed, err := url.Parse(address) + if err != nil { + return AlertmanagerConfig{}, err + } + + scheme := parsed.Scheme + host := parsed.Host + for _, qType := range []dns.QType{dns.A, dns.SRV, dns.SRVNoA} { + prefix := string(qType) + "+" + if strings.HasPrefix(strings.ToLower(scheme), prefix) { + // Scheme is of the form "+". + scheme = strings.TrimPrefix(scheme, prefix) + host = prefix + parsed.Host + if qType == dns.A { + if _, _, err := net.SplitHostPort(parsed.Host); err != nil { + // The host port could be missing. Append the defaultAlertmanagerPort. + host = host + ":" + strconv.Itoa(defaultAlertmanagerPort) + } + } + break + } + } + var basicAuth BasicAuth + if parsed.User != nil && parsed.User.String() != "" { + basicAuth.Username = parsed.User.Username() + pw, _ := parsed.User.Password() + basicAuth.Password = pw + } + + return AlertmanagerConfig{ + PathPrefix: parsed.Path, + Scheme: scheme, + StaticAddresses: []string{host}, + Timeout: model.Duration(timeout), + HTTPClientConfig: HTTPClientConfig{ + BasicAuth: basicAuth, + }, + }, nil +} + +// Endpoints returns the list of known Alertmanager endpoints. +func (a *Alertmanager) Endpoints() []*url.URL { + var urls []*url.URL + for _, addr := range a.provider.Addresses() { + urls = append(urls, + &url.URL{ + Scheme: a.scheme, + Host: addr, + Path: path.Join("/", a.prefix, alertPushEndpoint), + }, + ) + } + return urls +} + +// Do sends a POST request to the given URL. +func (a *Alertmanager) Do(ctx context.Context, u *url.URL, r io.Reader) error { + req, err := http.NewRequest("POST", u.String(), r) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, a.timeout) + defer cancel() + req = req.WithContext(ctx) + req.Header.Set("Content-Type", contentTypeJSON) + req.Header.Set("User-Agent", userAgent) + + resp, err := a.client.Do(req) + if err != nil { + return errors.Wrapf(err, "send request to %q", u) + } + defer runutil.ExhaustCloseWithLogOnErr(a.logger, resp.Body, "send one alert") + + if resp.StatusCode/100 != 2 { + return errors.Errorf("bad response status %v from %q", resp.Status, u) + } + return nil +} + +// Discover runs the service to discover target endpoints. +func (a *Alertmanager) Discover(ctx context.Context) { + var wg sync.WaitGroup + ch := make(chan []*targetgroup.Group) + + for _, d := range a.fileDiscoverers { + wg.Add(1) + go func(d *file.Discovery) { + d.Run(ctx, ch) + wg.Done() + }(d) + } + + func() { + for { + select { + case update := <-ch: + // Discoverers sometimes send nil updates so need to check for it to avoid panics. + if update == nil { + continue + } + a.fileSDCache.Update(update) + case <-ctx.Done(): + return + } + } + }() + wg.Wait() +} + +// Resolve refreshes and resolves the list of Alertmanager targets. +func (a *Alertmanager) Resolve(ctx context.Context) { + a.provider.Resolve(ctx, append(a.fileSDCache.Addresses(), a.staticAddresses...)) +} diff --git a/pkg/alert/client_test.go b/pkg/alert/client_test.go new file mode 100644 index 0000000000..8b29d51c9e --- /dev/null +++ b/pkg/alert/client_test.go @@ -0,0 +1,88 @@ +package alert + +import ( + "testing" + "time" + + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestBuildAlertmanagerConfiguration(t *testing.T) { + for _, tc := range []struct { + address string + + err bool + expected AlertmanagerConfig + }{ + { + address: "http://localhost:9093", + expected: AlertmanagerConfig{ + StaticAddresses: []string{"localhost:9093"}, + Scheme: "http", + }, + }, + { + address: "https://am.example.com", + expected: AlertmanagerConfig{ + StaticAddresses: []string{"am.example.com"}, + Scheme: "https", + }, + }, + { + address: "dns+http://localhost:9093", + expected: AlertmanagerConfig{ + StaticAddresses: []string{"dns+localhost:9093"}, + Scheme: "http", + }, + }, + { + address: "dnssrv+http://localhost", + expected: AlertmanagerConfig{ + StaticAddresses: []string{"dnssrv+localhost"}, + Scheme: "http", + }, + }, + { + address: "ssh+http://localhost", + expected: AlertmanagerConfig{ + StaticAddresses: []string{"localhost"}, + Scheme: "ssh+http", + }, + }, + { + address: "dns+https://localhost/path/prefix/", + expected: AlertmanagerConfig{ + StaticAddresses: []string{"dns+localhost:9093"}, + Scheme: "https", + PathPrefix: "/path/prefix/", + }, + }, + { + address: "http://user:pass@localhost:9093", + expected: AlertmanagerConfig{ + HTTPClientConfig: HTTPClientConfig{ + BasicAuth: BasicAuth{ + Username: "user", + Password: "pass", + }, + }, + StaticAddresses: []string{"localhost:9093"}, + Scheme: "http", + }, + }, + { + address: "://user:pass@localhost:9093", + err: true, + }, + } { + t.Run(tc.address, func(t *testing.T) { + cfg, err := BuildAlertmanagerConfig(nil, tc.address, time.Duration(0)) + if tc.err { + testutil.NotOk(t, err) + return + } + + testutil.Equals(t, tc.expected, cfg) + }) + } +} diff --git a/pkg/discovery/dns/provider.go b/pkg/discovery/dns/provider.go index 148143ea4d..332135ba17 100644 --- a/pkg/discovery/dns/provider.go +++ b/pkg/discovery/dns/provider.go @@ -76,6 +76,18 @@ func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType Reso return p } +// Clone returns a new provider from an existing one. +func (p *Provider) Clone() *Provider { + return &Provider{ + resolver: p.resolver, + resolved: make(map[string][]string), + logger: p.logger, + resolverAddrs: p.resolverAddrs, + resolverLookupsCount: p.resolverLookupsCount, + resolverFailuresCount: p.resolverFailuresCount, + } +} + // Resolve stores a list of provided addresses or their DNS records if requested. // Addresses prefixed with `dns+` or `dnssrv+` will be resolved through respective DNS lookup (A/AAAA or SRV). // defaultPort is used for non-SRV records when a port is not supplied. diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 9051848a38..2d5cec20e1 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -12,6 +12,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" + "github.com/thanos-io/thanos/pkg/alert" "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/cos" @@ -71,13 +72,20 @@ func main() { os.Exit(1) } } + + alertmgrCfg := alert.DefaultAlertmanagerConfig() + alertmgrCfg.FileSDConfigs = []alert.FileSDConfig{alert.FileSDConfig{}} + if err := generate(alert.AlertingConfig{Alertmanagers: []alert.AlertmanagerConfig{alertmgrCfg}}, "rule_alerting", *outputDir); err != nil { + level.Error(logger).Log("msg", "failed to generate", "type", "rule_alerting", "err", err) + os.Exit(1) + } logger.Log("msg", "success") } func generate(obj interface{}, typ string, outputDir string) error { // We forbid omitempty option. This is for simplification for doc generation. if err := checkForOmitEmptyTagOption(obj); err != nil { - return err + return errors.Wrap(err, "invalid type") } out, err := yaml.Marshal(obj) @@ -95,15 +103,15 @@ func checkForOmitEmptyTagOption(obj interface{}) error { func checkForOmitEmptyTagOptionRec(v reflect.Value) error { switch v.Kind() { case reflect.Struct: - for i := 0; i < v.NumField(); i += 1 { + for i := 0; i < v.NumField(); i++ { tags, err := structtag.Parse(string(v.Type().Field(i).Tag)) if err != nil { - return err + return errors.Wrapf(err, "%s: failed to parse tag %q", v.Type().Field(i).Name, v.Type().Field(i).Tag) } tag, err := tags.Get("yaml") if err != nil { - return err + return errors.Wrapf(err, "%s: failed to get tag %q", v.Type().Field(i).Name, v.Type().Field(i).Tag) } for _, opts := range tag.Options { @@ -113,16 +121,12 @@ func checkForOmitEmptyTagOptionRec(v reflect.Value) error { } if err := checkForOmitEmptyTagOptionRec(v.Field(i)); err != nil { - return err + return errors.Wrapf(err, "%s", v.Type().Field(i).Name) } } case reflect.Ptr: - if !v.IsValid() { - return errors.New("nil pointers are not allowed in configuration.") - } - - return errors.New("nil pointers are not allowed in configuration.") + return errors.New("nil pointers are not allowed in configuration") case reflect.Interface: return checkForOmitEmptyTagOptionRec(v.Elem()) diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 5784938aa8..7cfe508f53 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -1,22 +1,30 @@ package e2e_test import ( + "bytes" "context" "encoding/json" + "encoding/pem" "fmt" "io/ioutil" "math" "net/http" + "net/http/httptest" "os" "path/filepath" "sort" + "sync" "testing" "time" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" + yaml "gopkg.in/yaml.v2" + + "github.com/thanos-io/thanos/pkg/alert" "github.com/thanos-io/thanos/pkg/promclient" rapi "github.com/thanos-io/thanos/pkg/rule/api" "github.com/thanos-io/thanos/pkg/runutil" @@ -62,10 +70,314 @@ func createRuleFiles(t *testing.T, dir string) { } } +func serializeAlertingConfiguration(t *testing.T, cfg ...alert.AlertmanagerConfig) []byte { + t.Helper() + amCfg := alert.AlertingConfig{ + Alertmanagers: cfg, + } + b, err := yaml.Marshal(&amCfg) + if err != nil { + t.Errorf("failed to serialize alerting configuration: %v", err) + } + return b +} + +type mockAlertmanager struct { + path string + token string + mtx sync.Mutex + alerts []*model.Alert + lastError error +} + +func newMockAlertmanager(path string, token string) *mockAlertmanager { + return &mockAlertmanager{ + path: path, + token: token, + alerts: make([]*model.Alert, 0), + } +} + +func (m *mockAlertmanager) setLastError(err error) { + m.mtx.Lock() + defer m.mtx.Unlock() + m.lastError = err +} + +func (m *mockAlertmanager) LastError() error { + m.mtx.Lock() + defer m.mtx.Unlock() + return m.lastError +} + +func (m *mockAlertmanager) Alerts() []*model.Alert { + m.mtx.Lock() + defer m.mtx.Unlock() + return m.alerts +} + +func (m *mockAlertmanager) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + if req.Method != "POST" { + m.setLastError(errors.Errorf("invalid method: %s", req.Method)) + resp.WriteHeader(http.StatusMethodNotAllowed) + return + } + + if req.URL.Path != m.path { + m.setLastError(errors.Errorf("invalid path: %s", req.URL.Path)) + resp.WriteHeader(http.StatusNotFound) + return + } + + if m.token != "" { + auth := req.Header.Get("Authorization") + if auth != fmt.Sprintf("Bearer %s", m.token) { + m.setLastError(errors.Errorf("invalid auth: %s", req.URL.Path)) + resp.WriteHeader(http.StatusForbidden) + return + } + } + + b, err := ioutil.ReadAll(req.Body) + if err != nil { + m.setLastError(err) + resp.WriteHeader(http.StatusInternalServerError) + return + } + + var alerts []*model.Alert + if err := json.Unmarshal(b, &alerts); err != nil { + m.setLastError(err) + resp.WriteHeader(http.StatusInternalServerError) + return + } + + m.mtx.Lock() + m.alerts = append(m.alerts, alerts...) + m.mtx.Unlock() +} + +// TestRuleAlertmanagerHTTPClient verifies that Thanos Ruler can send alerts to +// Alertmanager in various setups: +// * Plain HTTP. +// * HTTPS with custom CA. +// * API with a prefix. +// * API protected by bearer token authentication. +// +// Because Alertmanager supports HTTP only and no authentication, the test uses +// a mocked server instead of the "real" Alertmanager service. +// The other end-to-end tests exercise against the "real" Alertmanager +// implementation. +func TestRuleAlertmanagerHTTPClient(t *testing.T) { + a := newLocalAddresser() + + // Plain HTTP with a prefix. + handler1 := newMockAlertmanager("/prefix/api/v1/alerts", "") + srv1 := httptest.NewServer(handler1) + defer srv1.Close() + // HTTPS with authentication. + handler2 := newMockAlertmanager("/api/v1/alerts", "secret") + srv2 := httptest.NewTLSServer(handler2) + defer srv2.Close() + + // Write the server's certificate to disk for the alerting configuration. + tlsDir, err := ioutil.TempDir("", "tls") + defer os.RemoveAll(tlsDir) + testutil.Ok(t, err) + var out bytes.Buffer + err = pem.Encode(&out, &pem.Block{Type: "CERTIFICATE", Bytes: srv2.TLS.Certificates[0].Certificate[0]}) + testutil.Ok(t, err) + caFile := filepath.Join(tlsDir, "ca.crt") + err = ioutil.WriteFile(caFile, out.Bytes(), 0640) + testutil.Ok(t, err) + + amCfg := serializeAlertingConfiguration( + t, + alert.AlertmanagerConfig{ + StaticAddresses: []string{srv1.Listener.Addr().String()}, + Scheme: "http", + Timeout: model.Duration(time.Second), + PathPrefix: "/prefix/", + }, + alert.AlertmanagerConfig{ + HTTPClientConfig: alert.HTTPClientConfig{ + TLSConfig: alert.TLSConfig{ + CAFile: caFile, + }, + BearerToken: "secret", + }, + StaticAddresses: []string{srv2.Listener.Addr().String()}, + Scheme: "https", + Timeout: model.Duration(time.Second), + }, + ) + + rulesDir, err := ioutil.TempDir("", "rules") + defer os.RemoveAll(rulesDir) + testutil.Ok(t, err) + createRuleFiles(t, rulesDir) + + qAddr := a.New() + r := rule(a.New(), a.New(), rulesDir, amCfg, []address{qAddr}, nil) + q := querier(qAddr, a.New(), []address{r.GRPC}, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + exit, err := e2eSpinup(t, ctx, q, r) + if err != nil { + t.Errorf("spinup failed: %v", err) + cancel() + return + } + + defer func() { + cancel() + <-exit + }() + + testutil.Ok(t, runutil.Retry(5*time.Second, ctx.Done(), func() (err error) { + select { + case <-exit: + cancel() + return nil + default: + } + + for i, am := range []*mockAlertmanager{handler1, handler2} { + if len(am.Alerts()) == 0 { + return errors.Errorf("no alert received from handler%d, last error: %v", i, am.LastError()) + } + } + + return nil + })) +} + +func TestRuleAlertmanagerFileSD(t *testing.T) { + a := newLocalAddresser() + + am := alertManager(a.New()) + amDir, err := ioutil.TempDir("", "am") + defer os.RemoveAll(amDir) + testutil.Ok(t, err) + amCfg := serializeAlertingConfiguration( + t, + alert.AlertmanagerConfig{ + FileSDConfigs: []alert.FileSDConfig{ + alert.FileSDConfig{ + Files: []string{filepath.Join(amDir, "*.yaml")}, + RefreshInterval: model.Duration(time.Hour), + }, + }, + Scheme: "http", + Timeout: model.Duration(time.Second), + }, + ) + + rulesDir, err := ioutil.TempDir("", "rules") + defer os.RemoveAll(rulesDir) + testutil.Ok(t, err) + createRuleFiles(t, rulesDir) + + qAddr := a.New() + r := rule(a.New(), a.New(), rulesDir, amCfg, []address{qAddr}, nil) + q := querier(qAddr, a.New(), []address{r.GRPC}, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + exit, err := e2eSpinup(t, ctx, am, q, r) + if err != nil { + t.Errorf("spinup failed: %v", err) + cancel() + return + } + + defer func() { + cancel() + <-exit + }() + + // Wait for a couple of evaluations and make sure that Alertmanager didn't receive anything. + testutil.Ok(t, runutil.Retry(5*time.Second, ctx.Done(), func() (err error) { + select { + case <-exit: + cancel() + return nil + default: + } + + // The time series written for the firing alerting rule must be queryable. + res, warnings, err := promclient.QueryInstant(ctx, nil, urlParse(t, q.HTTP.URL()), "max(count_over_time(ALERTS[1m])) > 2", time.Now(), promclient.QueryOptions{ + Deduplicate: false, + }) + if err != nil { + return err + } + if len(warnings) > 0 { + return errors.Errorf("unexpected warnings %s", warnings) + } + if len(res) == 0 { + return errors.Errorf("empty result") + } + + alrts, err := queryAlertmanagerAlerts(ctx, am.HTTP.URL()) + if err != nil { + return err + } + if len(alrts) != 0 { + return errors.Errorf("unexpected alerts length %d", len(alrts)) + } + + return nil + })) + + // Add the Alertmanager address to the file SD directory. + fileSDPath := filepath.Join(amDir, "targets.yaml") + b, err := yaml.Marshal([]*targetgroup.Group{ + &targetgroup.Group{ + Targets: []model.LabelSet{ + model.LabelSet{ + model.LabelName(model.AddressLabel): model.LabelValue(am.HTTP.HostPort()), + }, + }, + }, + }) + testutil.Ok(t, err) + + testutil.Ok(t, ioutil.WriteFile(fileSDPath+".tmp", b, 0660)) + testutil.Ok(t, os.Rename(fileSDPath+".tmp", fileSDPath)) + + // Verify that alerts are received by Alertmanager. + testutil.Ok(t, runutil.Retry(5*time.Second, ctx.Done(), func() (err error) { + select { + case <-exit: + cancel() + return nil + default: + } + alrts, err := queryAlertmanagerAlerts(ctx, am.HTTP.URL()) + if err != nil { + return err + } + if len(alrts) == 0 { + return errors.Errorf("expecting alerts") + } + + return nil + })) +} + func TestRule(t *testing.T) { a := newLocalAddresser() am := alertManager(a.New()) + amCfg := serializeAlertingConfiguration( + t, + alert.AlertmanagerConfig{ + StaticAddresses: []string{am.HTTP.HostPort()}, + Scheme: "http", + Timeout: model.Duration(time.Second), + }, + ) + qAddr := a.New() rulesDir, err := ioutil.TempDir("", "rules") @@ -73,8 +385,8 @@ func TestRule(t *testing.T) { testutil.Ok(t, err) createRuleFiles(t, rulesDir) - r1 := rule(a.New(), a.New(), rulesDir, am.HTTP, []address{qAddr}, nil) - r2 := rule(a.New(), a.New(), rulesDir, am.HTTP, nil, []address{qAddr}) + r1 := rule(a.New(), a.New(), rulesDir, amCfg, []address{qAddr}, nil) + r2 := rule(a.New(), a.New(), rulesDir, amCfg, nil, []address{qAddr}) q := querier(qAddr, a.New(), []address{r1.GRPC, r2.GRPC}, nil) @@ -282,19 +594,25 @@ func (a *failingStoreAPI) LabelValues(context.Context, *storepb.LabelValuesReque // Test Ruler behaviour on different storepb.PartialResponseStrategy when having partial response from single `failingStoreAPI`. func TestRulePartialResponse(t *testing.T) { - dir, err := ioutil.TempDir("", "test_rulepartial_response") - testutil.Ok(t, err) - defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - a := newLocalAddresser() qAddr := a.New() f := fakeStoreAPI(a.New(), &failingStoreAPI{}) am := alertManager(a.New()) + amCfg := serializeAlertingConfiguration( + t, + alert.AlertmanagerConfig{ + StaticAddresses: []string{am.HTTP.HostPort()}, + Scheme: "http", + Timeout: model.Duration(time.Second), + }, + ) + rulesDir, err := ioutil.TempDir("", "rules") defer os.RemoveAll(rulesDir) testutil.Ok(t, err) - r := rule(a.New(), a.New(), rulesDir, am.HTTP, []address{qAddr}, nil) + + r := rule(a.New(), a.New(), rulesDir, amCfg, []address{qAddr}, nil) q := querier(qAddr, a.New(), []address{r.GRPC, f.GRPC}, nil) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) diff --git a/test/e2e/spinup_test.go b/test/e2e/spinup_test.go index 4eea79e5fc..721eb70403 100644 --- a/test/e2e/spinup_test.go +++ b/test/e2e/spinup_test.go @@ -305,7 +305,7 @@ receivers: } } -func rule(http, grpc address, ruleDir string, am address, queryAddresses []address, queryFileSDAddresses []address) *serverScheduler { +func rule(http, grpc address, ruleDir string, amCfg []byte, queryAddresses []address, queryFileSDAddresses []address) *serverScheduler { return &serverScheduler{ HTTP: http, GRPC: grpc, @@ -317,12 +317,14 @@ func rule(http, grpc address, ruleDir string, am address, queryAddresses []addre "--data-dir", filepath.Join(workDir, "data"), "--rule-file", filepath.Join(ruleDir, "*.yaml"), "--eval-interval", "1s", - "--alertmanagers.url", am.URL(), + "--alertmanagers.config", string(amCfg), + "--alertmanagers.sd-dns-interval", "5s", "--grpc-address", grpc.HostPort(), "--grpc-grace-period", "0s", "--http-address", http.HostPort(), "--log.level", "debug", "--query.sd-dns-interval", "5s", + "--resend-delay", "5s", } for _, addr := range queryAddresses { From de869782f2624f7a9f22171649161f251a77ac04 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Wed, 18 Dec 2019 19:45:33 +0100 Subject: [PATCH 124/257] Fixed the gate duration metric unit (#1909) Signed-off-by: Marco Pracucci --- CHANGELOG.md | 1 + pkg/store/gate.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4634cb1dd..7652cc555f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1856](https://github.com/thanos-io/thanos/pull/1856) Receive: close DBReadOnly after flushing to fix a memory leak. - [#1882](https://github.com/thanos-io/thanos/pull/1882) Receive: upload to object storage as 'receive' rather than 'sidecar'. +- [#1907](https://github.com/thanos-io/thanos/pull/1907) Store: Fixed the duration unit for the metric `thanos_bucket_store_series_gate_duration_seconds`. ### Added - [#1852](https://github.com/thanos-io/thanos/pull/1852) Add support for `AWS_CONTAINER_CREDENTIALS_FULL_URI` by upgrading to minio-go v6.0.44 diff --git a/pkg/store/gate.go b/pkg/store/gate.go index 526a3e39a7..7662e5392f 100644 --- a/pkg/store/gate.go +++ b/pkg/store/gate.go @@ -41,7 +41,7 @@ func NewGate(maxConcurrent int, reg prometheus.Registerer) *Gate { func (g *Gate) IsMyTurn(ctx context.Context) error { start := time.Now() defer func() { - g.gateTiming.Observe(float64(time.Since(start))) + g.gateTiming.Observe(time.Since(start).Seconds()) }() if err := g.g.Start(ctx); err != nil { From dd934ec2697308753d8c33e7c5e0dab97f7d2f3e Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 19 Dec 2019 15:56:11 +0800 Subject: [PATCH 125/257] add go vet to lint (#1905) * add go vet to lint Signed-off-by: Xiang Dai <764524258@qq.com> * specify dir to skip vendor dir Signed-off-by: Xiang Dai <764524258@qq.com> --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 729a50724a..c7e14503bf 100644 --- a/Makefile +++ b/Makefile @@ -295,6 +295,8 @@ web: web-pre-process $(HUGO) # # to debug big allocations during linting. lint: check-git $(GOLANGCILINT) $(MISSPELL) + @echo ">> examining all of the Go files" + @go vet -stdmethods=false ./pkg/... ./cmd/... && go vet doc.go @echo ">> linting all of the Go files GOGC=${GOGC}" @$(GOLANGCILINT) run @echo ">> detecting misspells" From f5b7bacf762197c5fa8ec728257bbca264072e35 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 19 Dec 2019 11:39:41 +0100 Subject: [PATCH 126/257] cmd/thanos/rule: remove unused metric (#1912) Signed-off-by: Simon Pasquier --- cmd/thanos/rule.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 9e3af8c8d4..202aa8478a 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -234,10 +234,6 @@ func runRule( Name: "thanos_rule_duplicated_query_address", Help: "The number of times a duplicated query addresses is detected from the different configs in rule", }) - alertMngrAddrResolutionErrors := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "thanos_rule_alertmanager_address_resolution_errors", - Help: "The number of times resolving an address of an alertmanager has failed inside Thanos Rule", - }) rulesLoaded := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "thanos_rule_loaded_rules", @@ -257,7 +253,6 @@ func runRule( reg.MustRegister(configSuccess) reg.MustRegister(configSuccessTime) reg.MustRegister(duplicatedQuery) - reg.MustRegister(alertMngrAddrResolutionErrors) reg.MustRegister(rulesLoaded) reg.MustRegister(ruleEvalWarnings) From b32108b27156404d586c2cbe96e0230baa2bc0a5 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 19 Dec 2019 16:51:17 +0100 Subject: [PATCH 127/257] *: factorize HTTP client code into separate package (#1913) Signed-off-by: Simon Pasquier --- pkg/alert/client.go | 85 ++-------------------------- pkg/alert/client_test.go | 5 +- pkg/http/http.go | 116 +++++++++++++++++++++++++++++++++++++++ test/e2e/rule_test.go | 5 +- 4 files changed, 127 insertions(+), 84 deletions(-) create mode 100644 pkg/http/http.go diff --git a/pkg/alert/client.go b/pkg/alert/client.go index f58b995a2b..679f0f7f2e 100644 --- a/pkg/alert/client.go +++ b/pkg/alert/client.go @@ -2,7 +2,6 @@ package alert import ( "context" - "fmt" "io" "net" "net/http" @@ -15,15 +14,14 @@ import ( "github.com/go-kit/kit/log" "github.com/pkg/errors" - config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" - "github.com/prometheus/common/version" "github.com/prometheus/prometheus/discovery/file" "github.com/prometheus/prometheus/discovery/targetgroup" "gopkg.in/yaml.v2" "github.com/thanos-io/thanos/pkg/discovery/cache" "github.com/thanos-io/thanos/pkg/discovery/dns" + http_util "github.com/thanos-io/thanos/pkg/http" "github.com/thanos-io/thanos/pkg/runutil" ) @@ -33,8 +31,6 @@ const ( contentTypeJSON = "application/json" ) -var userAgent = fmt.Sprintf("Thanos/%s", version.Version) - type AlertingConfig struct { Alertmanagers []AlertmanagerConfig `yaml:"alertmanagers"` } @@ -43,7 +39,7 @@ type AlertingConfig struct { // TODO(simonpasquier): add support for API version (v1 or v2). type AlertmanagerConfig struct { // HTTP client configuration. - HTTPClientConfig HTTPClientConfig `yaml:"http_config"` + HTTPClientConfig http_util.ClientConfig `yaml:"http_config"` // List of addresses with DNS prefixes. StaticAddresses []string `yaml:"static_configs"` @@ -60,72 +56,6 @@ type AlertmanagerConfig struct { Timeout model.Duration `yaml:"timeout"` } -type HTTPClientConfig struct { - // The HTTP basic authentication credentials for the targets. - BasicAuth BasicAuth `yaml:"basic_auth"` - // The bearer token for the targets. - BearerToken string `yaml:"bearer_token"` - // The bearer token file for the targets. - BearerTokenFile string `yaml:"bearer_token_file"` - // HTTP proxy server to use to connect to the targets. - ProxyURL string `yaml:"proxy_url"` - // TLSConfig to use to connect to the targets. - TLSConfig TLSConfig `yaml:"tls_config"` -} - -type TLSConfig struct { - // The CA cert to use for the targets. - CAFile string `yaml:"ca_file"` - // The client cert file for the targets. - CertFile string `yaml:"cert_file"` - // The client key file for the targets. - KeyFile string `yaml:"key_file"` - // Used to verify the hostname for the targets. - ServerName string `yaml:"server_name"` - // Disable target certificate validation. - InsecureSkipVerify bool `yaml:"insecure_skip_verify"` -} - -type BasicAuth struct { - Username string `yaml:"username"` - Password string `yaml:"password"` - PasswordFile string `yaml:"password_file"` -} - -func (b BasicAuth) IsZero() bool { - return b.Username == "" && b.Password == "" && b.PasswordFile == "" -} - -func (c HTTPClientConfig) convert() (config_util.HTTPClientConfig, error) { - httpClientConfig := config_util.HTTPClientConfig{ - BearerToken: config_util.Secret(c.BearerToken), - BearerTokenFile: c.BearerTokenFile, - TLSConfig: config_util.TLSConfig{ - CAFile: c.TLSConfig.CAFile, - CertFile: c.TLSConfig.CertFile, - KeyFile: c.TLSConfig.KeyFile, - ServerName: c.TLSConfig.ServerName, - InsecureSkipVerify: c.TLSConfig.InsecureSkipVerify, - }, - } - if c.ProxyURL != "" { - var proxy config_util.URL - err := yaml.Unmarshal([]byte(c.ProxyURL), &proxy) - if err != nil { - return httpClientConfig, err - } - httpClientConfig.ProxyURL = proxy - } - if !c.BasicAuth.IsZero() { - httpClientConfig.BasicAuth = &config_util.BasicAuth{ - Username: c.BasicAuth.Username, - Password: config_util.Secret(c.BasicAuth.Password), - PasswordFile: c.BasicAuth.PasswordFile, - } - } - return httpClientConfig, httpClientConfig.Validate() -} - type FileSDConfig struct { Files []string `yaml:"files"` RefreshInterval model.Duration `yaml:"refresh_interval"` @@ -187,11 +117,7 @@ func NewAlertmanager(logger log.Logger, cfg AlertmanagerConfig, provider Address logger = log.NewNopLogger() } - httpClientConfig, err := cfg.HTTPClientConfig.convert() - if err != nil { - return nil, err - } - client, err := config_util.NewClientFromConfig(httpClientConfig, "alertmanager", false) + client, err := http_util.NewClient(cfg.HTTPClientConfig, "alertmanager") if err != nil { return nil, err } @@ -250,7 +176,7 @@ func BuildAlertmanagerConfig(logger log.Logger, address string, timeout time.Dur break } } - var basicAuth BasicAuth + var basicAuth http_util.BasicAuth if parsed.User != nil && parsed.User.String() != "" { basicAuth.Username = parsed.User.Username() pw, _ := parsed.User.Password() @@ -262,7 +188,7 @@ func BuildAlertmanagerConfig(logger log.Logger, address string, timeout time.Dur Scheme: scheme, StaticAddresses: []string{host}, Timeout: model.Duration(timeout), - HTTPClientConfig: HTTPClientConfig{ + HTTPClientConfig: http_util.ClientConfig{ BasicAuth: basicAuth, }, }, nil @@ -293,7 +219,6 @@ func (a *Alertmanager) Do(ctx context.Context, u *url.URL, r io.Reader) error { defer cancel() req = req.WithContext(ctx) req.Header.Set("Content-Type", contentTypeJSON) - req.Header.Set("User-Agent", userAgent) resp, err := a.client.Do(req) if err != nil { diff --git a/pkg/alert/client_test.go b/pkg/alert/client_test.go index 8b29d51c9e..d83ca7c275 100644 --- a/pkg/alert/client_test.go +++ b/pkg/alert/client_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/thanos-io/thanos/pkg/http" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -60,8 +61,8 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { { address: "http://user:pass@localhost:9093", expected: AlertmanagerConfig{ - HTTPClientConfig: HTTPClientConfig{ - BasicAuth: BasicAuth{ + HTTPClientConfig: http.ClientConfig{ + BasicAuth: http.BasicAuth{ Username: "user", Password: "pass", }, diff --git a/pkg/http/http.go b/pkg/http/http.go new file mode 100644 index 0000000000..8f096d9b84 --- /dev/null +++ b/pkg/http/http.go @@ -0,0 +1,116 @@ +// Package http is a wrapper around github.com/prometheus/common/config. +package http + +import ( + "fmt" + "net/http" + + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/version" + "gopkg.in/yaml.v2" +) + +// ClientConfig configures an HTTP client. +type ClientConfig struct { + // The HTTP basic authentication credentials for the targets. + BasicAuth BasicAuth `yaml:"basic_auth"` + // The bearer token for the targets. + BearerToken string `yaml:"bearer_token"` + // The bearer token file for the targets. + BearerTokenFile string `yaml:"bearer_token_file"` + // HTTP proxy server to use to connect to the targets. + ProxyURL string `yaml:"proxy_url"` + // TLSConfig to use to connect to the targets. + TLSConfig TLSConfig `yaml:"tls_config"` +} + +// TLSConfig configures TLS connections. +type TLSConfig struct { + // The CA cert to use for the targets. + CAFile string `yaml:"ca_file"` + // The client cert file for the targets. + CertFile string `yaml:"cert_file"` + // The client key file for the targets. + KeyFile string `yaml:"key_file"` + // Used to verify the hostname for the targets. + ServerName string `yaml:"server_name"` + // Disable target certificate validation. + InsecureSkipVerify bool `yaml:"insecure_skip_verify"` +} + +// BasicAuth configures basic authentication for HTTP clients. +type BasicAuth struct { + Username string `yaml:"username"` + Password string `yaml:"password"` + PasswordFile string `yaml:"password_file"` +} + +// IsZero returns false if basic authentication isn't enabled. +func (b BasicAuth) IsZero() bool { + return b.Username == "" && b.Password == "" && b.PasswordFile == "" +} + +// NewClient returns a new HTTP client. +func NewClient(cfg ClientConfig, name string) (*http.Client, error) { + httpClientConfig := config_util.HTTPClientConfig{ + BearerToken: config_util.Secret(cfg.BearerToken), + BearerTokenFile: cfg.BearerTokenFile, + TLSConfig: config_util.TLSConfig{ + CAFile: cfg.TLSConfig.CAFile, + CertFile: cfg.TLSConfig.CertFile, + KeyFile: cfg.TLSConfig.KeyFile, + ServerName: cfg.TLSConfig.ServerName, + InsecureSkipVerify: cfg.TLSConfig.InsecureSkipVerify, + }, + } + if cfg.ProxyURL != "" { + var proxy config_util.URL + err := yaml.Unmarshal([]byte(cfg.ProxyURL), &proxy) + if err != nil { + return nil, err + } + httpClientConfig.ProxyURL = proxy + } + if !cfg.BasicAuth.IsZero() { + httpClientConfig.BasicAuth = &config_util.BasicAuth{ + Username: cfg.BasicAuth.Username, + Password: config_util.Secret(cfg.BasicAuth.Password), + PasswordFile: cfg.BasicAuth.PasswordFile, + } + } + if err := httpClientConfig.Validate(); err != nil { + return nil, err + } + + client, err := config_util.NewClientFromConfig(httpClientConfig, name, false) + if err != nil { + return nil, err + } + client.Transport = &userAgentRoundTripper{name: userAgent, rt: client.Transport} + return client, nil +} + +var userAgent = fmt.Sprintf("Thanos/%s", version.Version) + +type userAgentRoundTripper struct { + name string + rt http.RoundTripper +} + +// RoundTrip implements the http.RoundTripper interface. +func (u userAgentRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + if r.UserAgent() == "" { + // The specification of http.RoundTripper says that it shouldn't mutate + // the request so make a copy of req.Header since this is all that is + // modified. + r2 := new(http.Request) + *r2 = *r + r2.Header = make(http.Header) + for k, s := range r.Header { + r2.Header[k] = s + } + r2.Header.Set("User-Agent", u.name) + r = r2 + } + return u.rt.RoundTrip(r) +} diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 7cfe508f53..66b7a01870 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -25,6 +25,7 @@ import ( yaml "gopkg.in/yaml.v2" "github.com/thanos-io/thanos/pkg/alert" + http_util "github.com/thanos-io/thanos/pkg/http" "github.com/thanos-io/thanos/pkg/promclient" rapi "github.com/thanos-io/thanos/pkg/rule/api" "github.com/thanos-io/thanos/pkg/runutil" @@ -200,8 +201,8 @@ func TestRuleAlertmanagerHTTPClient(t *testing.T) { PathPrefix: "/prefix/", }, alert.AlertmanagerConfig{ - HTTPClientConfig: alert.HTTPClientConfig{ - TLSConfig: alert.TLSConfig{ + HTTPClientConfig: http_util.ClientConfig{ + TLSConfig: http_util.TLSConfig{ CAFile: caFile, }, BearerToken: "secret", From 9888f8d11eed27bdd528352cbb7fb264498e5ca2 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 20 Dec 2019 11:58:15 +0000 Subject: [PATCH 128/257] Simplify running tests with excluding certain providers. (#1914) Still by default run all. Making it more difficult to skip makes sure we don't forget to run those. Signed-off-by: Bartek Plotka --- CONTRIBUTING.md | 9 ++--- Makefile | 19 +++------- docs/storage.md | 7 ++-- pkg/objstore/objtesting/foreach.go | 58 +++++++++++++++--------------- 4 files changed, 40 insertions(+), 53 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b6b6962505..9ea038e5b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,13 +51,8 @@ $ make build $ $ git push origin ``` -5. If you don't have a live object store ready add these envvars to skip tests for these: -- THANOS_SKIP_GCS_TESTS to skip GCS tests. -- THANOS_SKIP_S3_AWS_TESTS to skip AWS tests. -- THANOS_SKIP_AZURE_TESTS to skip Azure tests. -- THANOS_SKIP_SWIFT_TESTS to skip SWIFT tests. -- THANOS_SKIP_TENCENT_COS_TESTS to skip Tencent COS tests. -- THANOS_SKIP_ALIYUN_OSS_TESTS to skip Aliyun OSS tests. +5. If you don't have a live object store ready add this envvar to skip tests for these: +- THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS If you skip all of these, the store specific tests will be run against memory object storage only. CI runs GCS and inmem tests only for now. Not having these variables will produce auth errors against GCS, AWS, Azure or COS tests. diff --git a/Makefile b/Makefile index c7e14503bf..dd30ac178b 100644 --- a/Makefile +++ b/Makefile @@ -226,28 +226,19 @@ test: check-git install-deps @go install github.com/thanos-io/thanos/cmd/thanos # Be careful on GOCACHE. Those tests are sometimes using built Thanos/Prometheus binaries directly. Don't cache those. @rm -rf ${GOCACHE} - @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_ALIYUN_OSS_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets" + @echo ">> running all tests. Do export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS if you want to skip e2e tests against all real store buckets. Current value: ${THANOS_TEST_OBJSTORE_SKIP}" @go test $(shell go list ./... | grep -v /vendor/); .PHONY: test-ci -test-ci: export THANOS_SKIP_AZURE_TESTS = true -test-ci: export THANOS_SKIP_SWIFT_TESTS = true -test-ci: export THANOS_SKIP_TENCENT_COS_TESTS = true -test-ci: export THANOS_SKIP_ALIYUN_OSS_TESTS = true +test-ci: export THANOS_TEST_OBJSTORE_SKIP=AZURE,SWIFT,COS,ALIYUNOSS test-ci: - @echo ">> Skipping AZURE tests" - @echo ">> Skipping SWIFT tests" - @echo ">> Skipping TENCENT tests" - @echo ">> Skipping ALIYUN tests" + @echo ">> Skipping ${THANOS_TEST_OBJSTORE_SKIP} tests" $(MAKE) test .PHONY: test-local -test-local: export THANOS_SKIP_GCS_TESTS = true -test-local: export THANOS_SKIP_S3_AWS_TESTS = true +test-local: export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS test-local: - @echo ">> Skipping GCE tests" - @echo ">> Skipping S3 tests" - $(MAKE) test-ci + $(MAKE) test # install-deps installs dependencies for e2e tetss. # It installs supported versions of Prometheus and alertmanager to test against in e2e. diff --git a/docs/storage.md b/docs/storage.md index b0a2334b31..999e0b156f 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -157,8 +157,11 @@ Example working AWS IAM policy for user: (No bucket policy) To test the policy, set env vars for S3 access for *empty, not used* bucket as well as: -THANOS_SKIP_GCS_TESTS=true + +``` +THANOS_TEST_OBJSTORE_SKIP=GCS,AZURE,SWIFT,COS,ALIYUNOSS THANOS_ALLOW_EXISTING_BUCKET_USE=true +``` And run: `GOCACHE=off go test -v -run TestObjStore_AcceptanceTest_e2e ./pkg/...` @@ -190,7 +193,7 @@ We need access to CreateBucket and DeleteBucket and access to all buckets: } ``` -With this policy you should be able to run set `THANOS_SKIP_GCS_TESTS=true` and unset `S3_BUCKET` and run all tests using `make test`. +With this policy you should be able to run set `THANOS_TEST_OBJSTORE_SKIP=GCS,AZURE,SWIFT,COS,ALIYUNOSS` and unset `S3_BUCKET` and run all tests using `make test`. Details about AWS policies: https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go index 40db25838e..e96ac0ef5b 100644 --- a/pkg/objstore/objtesting/foreach.go +++ b/pkg/objstore/objtesting/foreach.go @@ -3,8 +3,10 @@ package objtesting import ( "io/ioutil" "os" + "strings" "testing" + "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/filesystem" "github.com/thanos-io/thanos/pkg/objstore" @@ -18,23 +20,38 @@ import ( "github.com/thanos-io/thanos/pkg/testutil" ) +// IsObjStoreSkipped returns true if given provider ID is found in THANOS_TEST_OBJSTORE_SKIP array delimited by comma e.g: +// THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS. +func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool { + if e, ok := os.LookupEnv("THANOS_TEST_OBJSTORE_SKIP"); ok { + obstores := strings.Split(e, ",") + for _, objstore := range obstores { + if objstore == string(provider) { + t.Logf("%s found in THANOS_TEST_OBJSTORE_SKIP array. Skipping.", provider) + return true + } + } + } + + return false +} + // ForeachStore runs given test using all available objstore implementations. // For each it creates a new bucket with a random name and a cleanup function // that deletes it after test was run. -// Use THANOS_SKIP__TESTS to skip explicitly certain tests. +// Use THANOS_TEST_OBJSTORE_SKIP to skip explicitly certain object storages. func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) { t.Parallel() - // Mandatory Inmem. + // Mandatory Inmem. Not parallel, to detect problem early. if ok := t.Run("inmem", func(t *testing.T) { - t.Parallel() testFn(t, inmem.NewBucket()) }); !ok { return } // Mandatory Filesystem. - if ok := t.Run("filesystem", func(t *testing.T) { + t.Run("filesystem", func(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "filesystem-foreach-store-test") @@ -44,12 +61,10 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) b, err := filesystem.NewBucket(dir) testutil.Ok(t, err) testFn(t, b) - }); !ok { - return - } + }) // Optional GCS. - if _, ok := os.LookupEnv("THANOS_SKIP_GCS_TESTS"); !ok { + if !IsObjStoreSkipped(t, client.GCS) { t.Run("gcs", func(t *testing.T) { bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT")) testutil.Ok(t, err) @@ -60,13 +75,10 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) // TODO(bwplotka): Add leaktest when https://github.com/GoogleCloudPlatform/google-cloud-go/issues/1025 is resolved. testFn(t, bkt) }) - - } else { - t.Log("THANOS_SKIP_GCS_TESTS envvar present. Skipping test against GCS.") } // Optional S3. - if _, ok := os.LookupEnv("THANOS_SKIP_S3_AWS_TESTS"); !ok { + if !IsObjStoreSkipped(t, client.S3) { t.Run("aws s3", func(t *testing.T) { // TODO(bwplotka): Allow taking location from envvar. bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2") @@ -81,13 +93,10 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) testFn(t, bkt) }) - - } else { - t.Log("THANOS_SKIP_S3_AWS_TESTS envvar present. Skipping test against S3 AWS.") } // Optional Azure. - if _, ok := os.LookupEnv("THANOS_SKIP_AZURE_TESTS"); !ok { + if !IsObjStoreSkipped(t, client.AZURE) { t.Run("azure", func(t *testing.T) { bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests") testutil.Ok(t, err) @@ -97,13 +106,10 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) testFn(t, bkt) }) - - } else { - t.Log("THANOS_SKIP_AZURE_TESTS envvar present. Skipping test against Azure.") } // Optional SWIFT. - if _, ok := os.LookupEnv("THANOS_SKIP_SWIFT_TESTS"); !ok { + if !IsObjStoreSkipped(t, client.SWIFT) { t.Run("swift", func(t *testing.T) { container, closeFn, err := swift.NewTestContainer(t) testutil.Ok(t, err) @@ -113,13 +119,10 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) testFn(t, container) }) - - } else { - t.Log("THANOS_SKIP_SWIFT_TESTS envvar present. Skipping test against swift.") } // Optional COS. - if _, ok := os.LookupEnv("THANOS_SKIP_TENCENT_COS_TESTS"); !ok { + if !IsObjStoreSkipped(t, client.COS) { t.Run("Tencent cos", func(t *testing.T) { bkt, closeFn, err := cos.NewTestBucket(t) testutil.Ok(t, err) @@ -129,13 +132,10 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) testFn(t, bkt) }) - - } else { - t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.") } // Optional OSS. - if _, ok := os.LookupEnv("THANOS_SKIP_ALIYUN_OSS_TESTS"); !ok { + if !IsObjStoreSkipped(t, client.ALIYUNOSS) { bkt, closeFn, err := oss.NewTestBucket(t) testutil.Ok(t, err) @@ -147,7 +147,5 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) if !ok { return } - } else { - t.Log("THANOS_SKIP_ALIYUN_OSS_TESTS envvar present. Skipping test against AliYun OSS.") } } From c2e338f72c9a909b45c2443f30f0243db2d38fdc Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 20 Dec 2019 15:05:27 +0100 Subject: [PATCH 129/257] Makefile: use gsed on OSX to keep GNU sed compatibility (#1916) Signed-off-by: Marco Pracucci --- Makefile | 16 ++++++++++------ scripts/cleanup-white-noise.sh | 4 +++- scripts/genflagdocs.sh | 3 ++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index dd30ac178b..6ebe3f0612 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,10 @@ JSONNET_BUNDLER ?= $(GOBIN)/jb-$(JSONNET_BUNDLER_VERSION) PROMTOOL_VERSION ?= edeb7a44cbf745f1d8be4ea6f215e79e651bfe19 PROMTOOL ?= $(GOBIN)/promtool-$(PROMTOOL_VERSION) +# Support gsed on OSX (installed via brew), falling back to sed. On Linux +# systems gsed won't be installed, so will use sed as expected. +SED ?= $(shell which gsed || which sed) + MIXIN_ROOT ?= mixin/thanos JSONNET_VENDOR_DIR ?= mixin/vendor @@ -168,8 +172,8 @@ docker-push: # docs regenerates flags in docs for all thanos commands. .PHONY: docs docs: $(EMBEDMD) build - @EMBEDMD_BIN="$(EMBEDMD)" scripts/genflagdocs.sh - @find -type f -name "*.md" | xargs scripts/cleanup-white-noise.sh + @EMBEDMD_BIN="$(EMBEDMD)" SED_BIN="$(SED)" scripts/genflagdocs.sh + @find . -type f -name "*.md" | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh # check-docs checks: # - discrepancy with flags is valid @@ -177,10 +181,10 @@ docs: $(EMBEDMD) build # - white noise .PHONY: check-docs check-docs: $(EMBEDMD) $(LICHE) build - @EMBEDMD_BIN="$(EMBEDMD)" scripts/genflagdocs.sh check + @EMBEDMD_BIN="$(EMBEDMD)" SED_BIN="$(SED)" scripts/genflagdocs.sh check @$(LICHE) --recursive docs --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root . @$(LICHE) --exclude "(cloud.tencent.com|goreportcard.com|alibabacloud.com)" --document-root . *.md - @find -type f -name "*.md" | xargs scripts/cleanup-white-noise.sh + @find . -type f -name "*.md" | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh $(call require_clean_work_tree,"check documentation") # checks Go code comments if they have trailing period (excludes protobuffers and vendor files). @@ -197,7 +201,7 @@ check-comments: format: $(GOIMPORTS) check-comments @echo ">> formatting code" @$(GOIMPORTS) -w $(FILES_TO_FMT) - @scripts/cleanup-white-noise.sh $(FILES_TO_FMT) + @SED_BIN="$(SED)" scripts/cleanup-white-noise.sh $(FILES_TO_FMT) # proto generates golang files from Thanos proto files. .PHONY: proto @@ -293,7 +297,7 @@ lint: check-git $(GOLANGCILINT) $(MISSPELL) @echo ">> detecting misspells" @find . -type f | grep -v vendor/ | grep -vE '\./\..*' | xargs $(MISSPELL) -error @echo ">> detecting white noise" - @find . -type f \( -name "*.md" -o -name "*.go" \) | xargs scripts/cleanup-white-noise.sh + @find . -type f \( -name "*.md" -o -name "*.go" \) | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh $(call require_clean_work_tree,"lint") .PHONY: web-serve diff --git a/scripts/cleanup-white-noise.sh b/scripts/cleanup-white-noise.sh index 8082a9376f..9cd366b33d 100755 --- a/scripts/cleanup-white-noise.sh +++ b/scripts/cleanup-white-noise.sh @@ -1,2 +1,4 @@ #!/bin/bash -sed -i 's/[ \t]*$//' "$@" +SED_BIN=${SED_BIN:-sed} + +${SED_BIN} -i 's/[ \t]*$//' "$@" diff --git a/scripts/genflagdocs.sh b/scripts/genflagdocs.sh index 278a7a4083..5393b311dc 100755 --- a/scripts/genflagdocs.sh +++ b/scripts/genflagdocs.sh @@ -5,6 +5,7 @@ set -e set -u EMBEDMD_BIN=${EMBEDMD_BIN:-embedmd} +SED_BIN=${SED_BIN:-sed} function docs { # if check arg was passed, instead of the docs generation verifies if docs coincide with the codebase @@ -50,7 +51,7 @@ for x in "${checkCommands[@]}"; do done # remove white noise -sed -i -e 's/[ \t]*$//' docs/components/flags/*.txt +${SED_BIN} -i -e 's/[ \t]*$//' docs/components/flags/*.txt go run scripts/cfggen/main.go --output-dir=docs/flags From ed6087f99af8b91813bb73dfa13266d14d5dd2c9 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Fri, 20 Dec 2019 10:31:05 -0500 Subject: [PATCH 130/257] support post in /api/v1/labels (#1910) Signed-off-by: yeya24 --- pkg/query/api/v1.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/query/api/v1.go b/pkg/query/api/v1.go index a3b97513ce..9dd9a9ec83 100644 --- a/pkg/query/api/v1.go +++ b/pkg/query/api/v1.go @@ -164,6 +164,7 @@ func (api *API) Register(r *route.Router, tracer opentracing.Tracer, logger log. r.Post("/series", instr("series", api.series)) r.Get("/labels", instr("label_names", api.labelNames)) + r.Post("/labels", instr("label_names", api.labelNames)) } type queryData struct { From 1b70eb4b5e59c84b466a0905383d0c35cd17b614 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Mon, 23 Dec 2019 17:23:30 +0800 Subject: [PATCH 131/257] Fix sed issue (#1925) Signed-off-by: Xiang Dai <764524258@qq.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6ebe3f0612..6bc8cb495f 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ PROMTOOL ?= $(GOBIN)/promtool-$(PROMTOOL_VERSION) # Support gsed on OSX (installed via brew), falling back to sed. On Linux # systems gsed won't be installed, so will use sed as expected. -SED ?= $(shell which gsed || which sed) +SED ?= $(shell which gsed 2>/dev/null || which sed) MIXIN_ROOT ?= mixin/thanos JSONNET_VENDOR_DIR ?= mixin/vendor From f308e57f1c3ea7e5a1f3b8c18f5fe27009a5b4f5 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 23 Dec 2019 10:52:19 +0000 Subject: [PATCH 132/257] Exclude couchdb.apache from link checking. (#1926) We don't check so often and it's very flaky on our CI. Signed-off-by: Bartek Plotka --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6bc8cb495f..ae67064ca2 100644 --- a/Makefile +++ b/Makefile @@ -182,8 +182,8 @@ docs: $(EMBEDMD) build .PHONY: check-docs check-docs: $(EMBEDMD) $(LICHE) build @EMBEDMD_BIN="$(EMBEDMD)" SED_BIN="$(SED)" scripts/genflagdocs.sh check - @$(LICHE) --recursive docs --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root . - @$(LICHE) --exclude "(cloud.tencent.com|goreportcard.com|alibabacloud.com)" --document-root . *.md + @$(LICHE) --recursive docs --exclude "(couchdb.apache.org/bylaws.html|cloud.tencent.com|alibabacloud.com)" --document-root . + @$(LICHE) --exclude "goreportcard.com" --document-root . *.md @find . -type f -name "*.md" | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh $(call require_clean_work_tree,"check documentation") From 687de93cdbed7918d39a40f4ba557b0076039277 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 23 Dec 2019 05:52:34 -0500 Subject: [PATCH 133/257] fix test cache addresses (#1921) Signed-off-by: yeya24 --- pkg/discovery/cache/cache_test.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/discovery/cache/cache_test.go b/pkg/discovery/cache/cache_test.go index cd40de760a..f1d6c1362d 100644 --- a/pkg/discovery/cache/cache_test.go +++ b/pkg/discovery/cache/cache_test.go @@ -2,6 +2,7 @@ package cache import ( "reflect" + "sort" "testing" "github.com/prometheus/common/model" @@ -12,16 +13,16 @@ func TestCacheAddresses(t *testing.T) { tgs := make(map[string]*targetgroup.Group) tgs["g1"] = &targetgroup.Group{ Targets: []model.LabelSet{ - model.LabelSet{model.AddressLabel: "localhost:9090"}, - model.LabelSet{model.AddressLabel: "localhost:9091"}, - model.LabelSet{model.AddressLabel: "localhost:9092"}, + {model.AddressLabel: "localhost:9090"}, + {model.AddressLabel: "localhost:9091"}, + {model.AddressLabel: "localhost:9092"}, }, } tgs["g2"] = &targetgroup.Group{ Targets: []model.LabelSet{ - model.LabelSet{model.AddressLabel: "localhost:9091"}, - model.LabelSet{model.AddressLabel: "localhost:9092"}, - model.LabelSet{model.AddressLabel: "localhost:9093"}, + {model.AddressLabel: "localhost:9091"}, + {model.AddressLabel: "localhost:9092"}, + {model.AddressLabel: "localhost:9093"}, }, } @@ -33,7 +34,12 @@ func TestCacheAddresses(t *testing.T) { "localhost:9092", "localhost:9093", } - if got := c.Addresses(); !reflect.DeepEqual(got, expected) { + + got := c.Addresses() + sort.Slice(got, func(i, j int) bool { + return i < j + }) + if !reflect.DeepEqual(got, expected) { t.Errorf("expected %v, want %v", got, expected) } } From 39f8623b577f6af660601ee29bd32b2600d3f356 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Mon, 23 Dec 2019 14:54:39 +0100 Subject: [PATCH 134/257] Introduce an internal option to enable vertical compaction (#1917) * Introduce an internal option to enable vertical compaction Signed-off-by: Marco Pracucci * Added metric with the count of compactions on overlapping blocks Signed-off-by: Marco Pracucci --- cmd/thanos/compact.go | 2 +- pkg/compact/compact.go | 64 ++++++++++++++++++++++++--------- pkg/compact/compact_e2e_test.go | 8 ++--- pkg/compact/compact_test.go | 2 +- 4 files changed, 53 insertions(+), 23 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 1a08135744..8d9c4e9222 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -226,7 +226,7 @@ func runCompact( }() sy, err := compact.NewSyncer(logger, reg, bkt, consistencyDelay, - blockSyncConcurrency, acceptMalformedIndex, relabelConfig) + blockSyncConcurrency, acceptMalformedIndex, false, relabelConfig) if err != nil { return errors.Wrap(err, "create syncer") } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 277904eeec..b3d1c32ac9 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -41,17 +41,18 @@ var blockTooFreshSentinelError = errors.New("Block too fresh") // Syncer syncronizes block metas from a bucket into a local directory. // It sorts them into compaction groups based on equal label sets. type Syncer struct { - logger log.Logger - reg prometheus.Registerer - bkt objstore.Bucket - consistencyDelay time.Duration - mtx sync.Mutex - blocks map[ulid.ULID]*metadata.Meta - blocksMtx sync.Mutex - blockSyncConcurrency int - metrics *syncerMetrics - acceptMalformedIndex bool - relabelConfig []*relabel.Config + logger log.Logger + reg prometheus.Registerer + bkt objstore.Bucket + consistencyDelay time.Duration + mtx sync.Mutex + blocks map[ulid.ULID]*metadata.Meta + blocksMtx sync.Mutex + blockSyncConcurrency int + metrics *syncerMetrics + acceptMalformedIndex bool + enableVerticalCompaction bool + relabelConfig []*relabel.Config } type syncerMetrics struct { @@ -66,6 +67,7 @@ type syncerMetrics struct { compactionRunsStarted *prometheus.CounterVec compactionRunsCompleted *prometheus.CounterVec compactionFailures *prometheus.CounterVec + verticalCompactions *prometheus.CounterVec } func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { @@ -119,6 +121,10 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { Name: "thanos_compact_group_compactions_failures_total", Help: "Total number of failed group compactions.", }, []string{"group"}) + m.verticalCompactions = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_compact_group_vertical_compactions_total", + Help: "Total number of group compaction attempts that resulted in a new block based on overlapping blocks.", + }, []string{"group"}) if reg != nil { reg.MustRegister( @@ -133,6 +139,7 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { m.compactionRunsStarted, m.compactionRunsCompleted, m.compactionFailures, + m.verticalCompactions, ) } return &m @@ -140,7 +147,7 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { // NewSyncer returns a new Syncer for the given Bucket and directory. // Blocks must be at least as old as the sync delay for being considered. -func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, consistencyDelay time.Duration, blockSyncConcurrency int, acceptMalformedIndex bool, relabelConfig []*relabel.Config) (*Syncer, error) { +func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, consistencyDelay time.Duration, blockSyncConcurrency int, acceptMalformedIndex bool, enableVerticalCompaction bool, relabelConfig []*relabel.Config) (*Syncer, error) { if logger == nil { logger = log.NewNopLogger() } @@ -154,6 +161,10 @@ func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket blockSyncConcurrency: blockSyncConcurrency, acceptMalformedIndex: acceptMalformedIndex, relabelConfig: relabelConfig, + // The syncer offers an option to enable vertical compaction, even if it's + // not currently used by Thanos, because the compactor is also used by Cortex + // which needs vertical compaction. + enableVerticalCompaction: enableVerticalCompaction, }, nil } @@ -366,10 +377,12 @@ func (c *Syncer) Groups() (res []*Group, err error) { labels.FromMap(m.Thanos.Labels), m.Thanos.Downsample.Resolution, c.acceptMalformedIndex, + c.enableVerticalCompaction, c.metrics.compactions.WithLabelValues(GroupKey(m.Thanos)), c.metrics.compactionRunsStarted.WithLabelValues(GroupKey(m.Thanos)), c.metrics.compactionRunsCompleted.WithLabelValues(GroupKey(m.Thanos)), c.metrics.compactionFailures.WithLabelValues(GroupKey(m.Thanos)), + c.metrics.verticalCompactions.WithLabelValues(GroupKey(m.Thanos)), c.metrics.garbageCollectedBlocks, ) if err != nil { @@ -514,10 +527,12 @@ type Group struct { mtx sync.Mutex blocks map[ulid.ULID]*metadata.Meta acceptMalformedIndex bool + enableVerticalCompaction bool compactions prometheus.Counter compactionRunsStarted prometheus.Counter compactionRunsCompleted prometheus.Counter compactionFailures prometheus.Counter + verticalCompactions prometheus.Counter groupGarbageCollectedBlocks prometheus.Counter } @@ -528,10 +543,12 @@ func newGroup( lset labels.Labels, resolution int64, acceptMalformedIndex bool, + enableVerticalCompaction bool, compactions prometheus.Counter, compactionRunsStarted prometheus.Counter, compactionRunsCompleted prometheus.Counter, compactionFailures prometheus.Counter, + verticalCompactions prometheus.Counter, groupGarbageCollectedBlocks prometheus.Counter, ) (*Group, error) { if logger == nil { @@ -544,10 +561,12 @@ func newGroup( resolution: resolution, blocks: map[ulid.ULID]*metadata.Meta{}, acceptMalformedIndex: acceptMalformedIndex, + enableVerticalCompaction: enableVerticalCompaction, compactions: compactions, compactionRunsStarted: compactionRunsStarted, compactionRunsCompleted: compactionRunsCompleted, compactionFailures: compactionFailures, + verticalCompactions: verticalCompactions, groupGarbageCollectedBlocks: groupGarbageCollectedBlocks, } return g, nil @@ -807,8 +826,13 @@ func (cg *Group) compact(ctx context.Context, dir string, comp tsdb.Compactor) ( defer cg.mtx.Unlock() // Check for overlapped blocks. + overlappingBlocks := false if err := cg.areBlocksOverlapping(nil); err != nil { - return false, ulid.ULID{}, halt(errors.Wrap(err, "pre compaction overlap check")) + if !cg.enableVerticalCompaction { + return false, ulid.ULID{}, halt(errors.Wrap(err, "pre compaction overlap check")) + } + + overlappingBlocks = true } // Planning a compaction works purely based on the meta.json files in our future group's dir. @@ -917,8 +941,11 @@ func (cg *Group) compact(ctx context.Context, dir string, comp tsdb.Compactor) ( return true, ulid.ULID{}, nil } cg.compactions.Inc() + if overlappingBlocks { + cg.verticalCompactions.Inc() + } level.Debug(cg.logger).Log("msg", "compacted blocks", - "blocks", fmt.Sprintf("%v", plan), "duration", time.Since(begin)) + "blocks", fmt.Sprintf("%v", plan), "duration", time.Since(begin), "overlapping_blocks", overlappingBlocks) bdir := filepath.Join(dir, compID.String()) index := filepath.Join(bdir, block.IndexFilename) @@ -942,9 +969,12 @@ func (cg *Group) compact(ctx context.Context, dir string, comp tsdb.Compactor) ( return false, ulid.ULID{}, halt(errors.Wrapf(err, "invalid result block %s", bdir)) } - // Ensure the output block is not overlapping with anything else. - if err := cg.areBlocksOverlapping(newMeta, plan...); err != nil { - return false, ulid.ULID{}, halt(errors.Wrapf(err, "resulted compacted block %s overlaps with something", bdir)) + // Ensure the output block is not overlapping with anything else, + // unless vertical compaction is enabled. + if !cg.enableVerticalCompaction { + if err := cg.areBlocksOverlapping(newMeta, plan...); err != nil { + return false, ulid.ULID{}, halt(errors.Wrapf(err, "resulted compacted block %s overlaps with something", bdir)) + } } if err := block.WriteIndexCache(cg.logger, index, indexCache); err != nil { diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index f0a4b9e139..0484aad4fd 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -37,7 +37,7 @@ func TestSyncer_SyncMetas_e2e(t *testing.T) { defer cancel() relabelConfig := make([]*relabel.Config, 0) - sy, err := NewSyncer(nil, nil, bkt, 0, 1, false, relabelConfig) + sy, err := NewSyncer(nil, nil, bkt, 0, 1, false, false, relabelConfig) testutil.Ok(t, err) // Generate 15 blocks. Initially the first 10 are synced into memory and only the last @@ -140,7 +140,7 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) { } // Do one initial synchronization with the bucket. - sy, err := NewSyncer(nil, nil, bkt, 0, 1, false, relabelConfig) + sy, err := NewSyncer(nil, nil, bkt, 0, 1, false, false, relabelConfig) testutil.Ok(t, err) testutil.Ok(t, sy.SyncMetas(ctx)) @@ -209,7 +209,7 @@ func TestGroup_Compact_e2e(t *testing.T) { reg := prometheus.NewRegistry() - sy, err := NewSyncer(logger, reg, bkt, 0*time.Second, 5, false, nil) + sy, err := NewSyncer(logger, reg, bkt, 0*time.Second, 5, false, false, nil) testutil.Ok(t, err) comp, err := tsdb.NewLeveledCompactor(ctx, reg, logger, []int64{1000, 3000}, nil) @@ -515,7 +515,7 @@ func TestSyncer_SyncMetasFilter_e2e(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() - sy, err := NewSyncer(nil, nil, bkt, 0, 1, false, relabelConfig) + sy, err := NewSyncer(nil, nil, bkt, 0, 1, false, false, relabelConfig) testutil.Ok(t, err) var ids []ulid.ULID diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 85d0f332fd..3624d27fda 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -79,7 +79,7 @@ func TestSyncer_SyncMetas_HandlesMalformedBlocks(t *testing.T) { bkt := inmem.NewBucket() relabelConfig := make([]*relabel.Config, 0) - sy, err := NewSyncer(nil, nil, bkt, 10*time.Second, 1, false, relabelConfig) + sy, err := NewSyncer(nil, nil, bkt, 10*time.Second, 1, false, false, relabelConfig) testutil.Ok(t, err) // Generate 1 block which is older than MinimumAgeForRemoval which has chunk data but no meta. Compactor should delete it. From 47c0ae2b33a8b41013390752632017953ffbcc0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=92=B1=E6=88=90=E9=BE=99?= Date: Thu, 2 Jan 2020 22:19:38 +0800 Subject: [PATCH 135/257] add tencent in adopters.yml (#1929) Signed-off-by: QianChenglong --- website/data/adopters.yml | 3 +++ website/static/logos/tencent.png | Bin 0 -> 10891 bytes 2 files changed, 3 insertions(+) create mode 100644 website/static/logos/tencent.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index 773fb387c1..5a41850880 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -66,3 +66,6 @@ adopters: - name: Grofers url: https://grofers.com logo: grofers.png +- name: Tencent + url: https://github.com/tkestack/tke + logo: tencent.png diff --git a/website/static/logos/tencent.png b/website/static/logos/tencent.png new file mode 100644 index 0000000000000000000000000000000000000000..c4b470672966bd38a0391150dad30f03876303e1 GIT binary patch literal 10891 zcmc(F1ydYdu=ZlXU4u(-m*DR11b25|+#$HT2X|+24ZZ}2;BEl|1b6uMz29%Rbx-wl zo#}bH`#F7js%EApT2)yF6^RfD005xM$x5mN08mN)9SATGOS|dlI{*L!P*v2Fl5tUd ze*1t7hyvJy(1<=hy}m*@fB+64h!J4l2C!)b*h1ugPHjN|>*j~o4}eX}e@44DfMp}V z4g|1kg@6gXdm$Riu?@-*gx~;rfB$%gM6ho84*@>z(?)cw1{lafESg{)K!iR$a5l{l8Zfpk z5Elpy7)UgmR#@8>SO^EpW(X1v64tgAl8(Tt5mHac1jZhsn*mnMg7KqY0{hsahM;U) zCyKov0*v%F>1n1bs5I)$m za*pEGp_DZVFl&^{n8fhtG%21Z@$NA#pU3m=rU>X`4F4I>x(>mQ?pz2Zj``QE+LzDD zWK8H4O#fWFVh9^37(J9vpX81mhKID1X(OR;53p%DZ)^|SwFA6=#S=F|?%xaExkY#B zuq>Y^@a$p;9rSNqt)4knNC#tkbU9QnQU>%<`}GF2t>ujFZ=ByBTtCM2Zlw(FV7qig zb!}u1@2X@?LY6@1-z%9iAsjbSJAVS%9p{>**xt>Tw>KzeR{{w{!ue)s%Q z-Fygj^XuZ%1DPc58}BApQ6<4OB`XQx6V^X!iqnUoag=h#9MR4O+>9}X_oJG{#^9~B zb69T0vXdSumYvyEjC)XtJUgD+cLeQjUz@$ZhWWGbEi2|uafXB(NxVclOm*J)Ow@U^ zxi7|+oYH8SF`)th?qrvp6K-iI`^vIjoZBG3yKLnKCkW&~;0=iT3R?TdN=O zH2zs9E=^Zh2msJX$VrN6dawOmd(qIaCWQY7NOniSaFVLRkOf6i$ZI(evtU8jNJ$ZW z^`wyKq-TQOZH~kDz@KfTt4v=y?J%g;63f?HUu!aqw&}Iy+!``zd_=%->4&HN*e( zy0mCADhu-|2*%v2NDN)Cis% z(0$nK#XF0jNj@62VrU(~Pf$v=a7n!!QVUbRNh3f%?qwz|ZPF73Hy+IXT+=*asv{YmCyIx78 z8F%ph#Ng`3jK$__!tb%CgEOMo-n7ws{%?md7Zg9$#UGC{RwB0w_5AlV*+S1X!Y|*1 z9Hzl{wChLD9dHP@-!xmh{K_i-_VDIlZNHdohc3z$BB6-a2FzPBVsh0sU3V9KR-u=I z>o$X>reK;1XLfkP{mFkreW2`%^noQMr6cqA%xW=OLQ}3Culn;^852d1YoN@JQ@*o@ z+8x~kR*Hb9vgK!tDq#CglXC?tHnCv+xrlYF2ALo~v5NA1oQ_+Wr_=ySJx1;6^KC1g zGjJlU%qM5QPaI^jHZ!@Q^_Ga8op_b^g}k~?n2n9xMm)9L*+tbPARw(`9OWz^uqhwE zJ&F)ZF(o;zCj^y~mw`#OuA@_lyqob*kTN4EkNXXfoomHXL;w54E{eK~`s-TUU*n5$ z0@cDd-Bk`lv4%}jebpa~^%E20fm=!y<=G(1kuII?-b@@_bq%XPp$lvK9XS07jrdmU zvd=93MUJZ|c1Wiyk1;z~P_vih0?h=g7F(v8S=dUDUd_ zmML+nmA@M;&kGwQJ0M}FfpJTL*Xa1)3cSv@>@FX&DZD1_9P^(Jedk8#r!-?a#uJ&W zS6u5rj&AZP)5`@sD#Yimd~o<8pQjt>c>WA^$i{9Z8Dxw2*B%ZS%yczRtF?RmTWK1+ zzwbTo4piXoEf@<$c{uAUhY@&F7%v_aamA_orGr>}7kjS3Od+#lA? zlm@3fQNymhQ+hEc~PY`qq(g`?-f8UU$Z@w7rD&)Ayt2*|+N<&Qqd)u7p+hK;!S$J!)D4|^yWjd?K z$_~gdZwh@^HL8y^tDuwkX&2?>aHIEv^lmvv`{1!g{BW2z?C4@}sWQDcwsuCt_MN2U zOk(f-&aU7JWgU^ZRL)?F|J%;Y1k1(bo%@CYf|Q)X_<@|rpR|C+u~HhUC}46fRgmC@ zVY*wBN6%X05%6gE#L4D2 zU&#oseOhpf!S6bH;yTp%8{ol-ilj;+82gkVZ;-!xse7`wx6ygZcLt4+)|ww0gv9#2 z+#%mGrIoNq?CdU{9DGM6AgRVSxbLSsl#*$)6mjz4lGDx;f^d=B(*`_LaR?*^&-8qRe1G{Jk-BgWJ8Gs;^SD& z+kS`>?+VZQ$vx7f(dx4ky>GYW)c!=6PJ*wg1BsjD6XPenM5B&~;-oUxasm3j$W?R0 z-C5pWqivx-jHm0k7-ATac|4L&Su&i?NvS;)QfX$*YQoJW%XCXe5Tf9DxCg&T%SyUJ zWkMnHCJe$20>z|CiU|@&safba63i3aKcV`@5$%89Kd>3y%(6*2t+vbYX{a{RDniA& zP{}#%y4>{ZzwEy31J0`!^X-0@r%Bi2C$|pfGTwZddE*jUl{3LnNc^{%rdBe+ZHOYr znce)gH$D!nob^Z2;)4BbU%^C{8mWT4>$Ir>)Yf{AgDJ`MCp3-FAsqj1+n-Mq$gPisTKAZjY1K*Fqz&OKM-OZc3$e84xo*d{b>>rQ)CjHa~JnS7iLQ7_gYMDgB>Jpg>q$)8xEPUA;pf2GQeJiY_xI1FlsGkP ztt>XTk;CsW@MwA4Ikt7|v_t1wNjVcrHu}rMv=O!*1RL(aS2>M|jIkN#peg>>w@tiD zr0xW3wtr21AggV&qPAD-1d|oFNI@uf1PdP2EpWH~+)?;@an#P=i4F%3T~n!bMk_8~ zagD2^GIq;NCdTt+TqHiP;&Lrs9#ciR2G#M^@?j=iawXyQywQ~-bJWBO)@YNd zRtj_*fEYN0wS}}C_oqecvbk7zG*ZD1Us>Dz@5xhExTxGyeya3|bJlw@hkmEYLwflP z_j@;mg&XazXk&uju8&R*H3duX1{wBPUU%5^2j=RPx<52UdD-LXEjf;zO=j>?M<^u2 zDcLI?e%Ln8>HB8zw70gtO0;Gy(rc98;yT7H*(p7GILT?#m3~mBrmk}ih}4-{cCV|+ z_KR4SWAcVF8rnFVHZits7{DOqe25m}_iL_1D8W&?c~r>$^5PhhL&0}e*=g;h*?p`U zoj;WLF3HO1C_Zdjsv}Qb%pTXk2cy+`;N2&(o;mRswy4qt&+|{O zeY?fhjJ0ng6`kv?rT!`hZ1q3ZL+i;poF~8jAnaggI~r_pC{bNR%3ppIdau0E<6w{^ z|B4p6^ffj0NB`Wwc0-_G`fZt97Et2b!h!!E()RZoMszaafalw}lA%a+QF|L+O-)V2 zzwUGQ7rkFiZ1EGgHkaXZY*A!z%p6rHF7WZZK!GsEL=!2paZ@ND* zX|8_q70n$7n7{gno_D!D-@1;FDwbatcvh6sSsUcll9B3BHF-s$N!}ksC42=Usf^$4 zoR(a><)J9rsyajxb-^ULy~F39-m8m8ad=pG9$oo%F8qK`CuGg%yg~6X>Pe@pLAR-g zPcK9$-yyFzQMNVO4-+Qn>@OA;{;j;z?Y*86lKu|~S9?;dKQPY2QBDwBUYY|;dxyO% zL#MZ|w43d**;PIhyM7dmgd7O%GIeM&D{?X-pTp{rV+gF(4se609LkZ^*0dc5YaAo=~>2kSMF&(x?ZzTmh=-xce9P7=>F zcoC6{On%R-#?{BK%Nko977ltXuTXD0QbnpY#%po`w>BGf?G0|H`z2R)c69NH67Poflt$zsK4XQO3s>DHNdD?JzXAm9e64jjyA$$k`)RVgw@^ur@gReYL3m)y{U|r&*&J82?I9|g$AZP^{nMFoPc(&h z55>8uCuYiW_a(o&nEMrgNi5j=79nFYy&=nh<>f8%8Ss6+8bI*ZIX| zj^e;Mm!jyOe<~F(c2go=Qail?2ZKWGmfXzeE|yNTJP!5d=sO!MwFJ}ugwiF7?CpI_ zlYDwE8gm^cW{hwWnm1ZrkyI$xx6x$E&`yK}O)oo_6Y9^1=EF~+2NgSLl9h#x(Y@4B z?9+x8_^XH(1QNuF1*J@X=g;HyTNq=6S@4KO2E8;S^o`ut#$%;yyk4M5cE^Jp!^sC+ z6ddRNERUHF(R@PmCCR{7IDKE>iCl0HG`T4!$z8Sh^OwuXPYruxekVmDf?*g6plnCF z%I{5i)NWK~qjBe0N(xi=MHi1w9-pdQtr(42BHiH)BS+N(hE_UU_^(?akJK_wN z#oq)_Ohy|3E%SJ)nFx~5vdDpTO_A7DZUwmC7gzqo3v{Q4kK8r`JAX=?E zc$Ww?R}x!Mj06-NZdJP*>M6<@%GbESU{`F}UJRS|!Zwe7989tZlv5zn3llHs)-KF+ zkZ%qW2}?skAsv}5*(9%sLh~csVy5b~mJ{Ks8|IB_AP9bcu~4XoMR-1_9A;cW!qV{Z zw2_AG{n4)QLPY7gey5W!{`j&|?W;}t=zxm3g=yzt1QNaf_B@-ui{kZ01jw+KUmC2R z!t5i2M;d0?5!h1LOztOSjQ}`WT(bMUtpu}lJC+ZVf`0I>r6UoxIrDBuub8^#b3~OQ z&*JYe8JpOh1$Bv0IGw@3cq2i5me8$ukyqc9%t~@DB_T(c@USSIbrLXXiMP14=6Q0* zo-dlMaqeQmSe;FVDjtu?2NBv-Rk^>Ye<4Cg`nJ*Wc*}x2zR0Js<4A+Fsdlkw+R6bh z2;9}o@Ox!K0Z9&Q_qpSD_$u7C|2Zc9ajvPK`$a>2ecXlg-8C)_{1+|a019ai5y$M6 zMxs&#{c!fdk2573Zs`QQ7Q;dNA|DIIlGhP}WGTQ4*$R+g(D+D60Tp@^kg8dDHOF{1RI~-b4O`aMv(J&g*S(`aC zoKFAhmz|)3*QRD$vmQOj?qdbq&nd*Ibl-dNVDz9!7;*A5rzyLre|h_*Z`-0CQXxitqh5Q>Qiwe{WdNpo%6=f?;t_SaImqSeOu|{<_N{j?OS9q!6`5}CAL0b05Ak8BO zmQ}R=^qPS-B!Q$}XVY*|QiINgU2T!WMb zw{H;%F@8!aQA%nzBLf5YhUofca&`IBBRF5q$%bw<_~KQYd{FcXI!ZTT0}C}hbooec zZWCV5{quB@RWE2@L-@|aZ^JJjZc0^_vvnELjZAJ{Bq<^v=Mopiw% zky10(6a(65+3a^A^8QfPX8c{X1q&8 z&%dv^wjppD@eXL+0c#YuU)@uYD7BTf`BDSsyOb2<8c~a9Li?)dzwGr7^@p21xQ4b)!Fp?OH!IN?gofOXNWA(gKvO^Pc_FpCf+Bk~f0<~9`M9ZYMI z6z8_U9Iv&Ywi-*tEW$IPX?YqTVNaxv=<$H25PtgV->*+1%1s{pqdEQpbE5d%LCtm{_4oy2Pf0c+&h#J7M^RiI zs3TBvD)yp-eGJk{^JX_+lG_2|yb(#h2)N)sted$R;)^M{6ZB|kNPuvro>xVN`=Y0y zO^1=rtdVXY;NH}SdeHID(%jRLnvkyZB{6=)fU>w%v@5>^A1T&8FNUg;&CjQq!U8>PlBQA^9Ih61ij_(1A(QHJU0Q`o=%oixC4?J@ z9(HH^%>T@SgbHJAV|Bi#8Ji|U<7bB$k2A??s1*2D4t;dYN8z}o4YDBf>P{uM_$;i7 zpNZI@4fjenZa|kjGh>LaLc?J|^O$^T#yGA&@xXDr7rY>wOOsy@RP?SADX~kJcfC|@)mWc9|iWp5OBsYDqeKSDQ-c0+&<4GUuyk5RHu}Go2^2WZukA5D)u$fgRtY`?fQ#);F2+QZp0861DjH7?!!Qe<4^}g154-CD zoAC(#&Y!N1_99*}?dcEro*L|x>8q<+0HcfNj_$RYy8-{|D`2tBlk`({{DSqJncoC= zOY0d8TAepxs#z?Ahiuq5Vnj*-e7NtqmzlOqeNSA!nE~GFzWd<&Yfn`s1wL8=K9}em zTJZHL)DSs|c043=&JVwd!HNg-1b`-yBir!S*yzZ~gHfaG-4=U{Gd2UXIRdbKTt~Q5@KznD34? z%^60mV95}gCq0`!H(?sA`mAV^FeWdRZ91;tYP?jyrd8oTD$*g8gsM-ynf}nxGy&zb z5DyyPCNUi`N&&>JVy!8>>Pan+)S~pKvm(PWIPiv2v;E&TLol9NQenkKKLIW3Ha;J$ zjPaU2!wlU0N;Rw+{L9~Q4ri3aVu6WRAC}n+2+F~6rByr=#)6yg3nb%UfL-O~xbRr; z{^E$Knxo%$`8-9;Eo~`>^jFKWH(KO!!xGG;<<12_*PQ%WDjw`7N~QbKQMX6gC@4rl zVQ|{>AzOw(X(d-3MPm+EDxNV!Lrz{v2DFbbg!L*w-giCoZ>6nLX5oF>D68eOunI42 zQsw8g{%EhloL*r;{s=Ih8XuSyEn@fV+wmF*mHTutf%32z&8$k=MmIaEW*hSRw7~@L ztT$v0QT-A`C;oZC<`7=@@=9){8xbG*lBRu0hVkeWBUKSqeBJ;fMbAe4M|ptV%Z!yJ zZ=}NfPWWwqgeNMhCv|y>sm4xGAxmsql9&Yo#i<+tMTegSI+HO4ZbmS12M3~A7NpsZ z>K~vGt~_>NfBifD%yJ&_Ge-7Z&iZd!*iq9*_L=Ed&iUh+AoObsvc)gBz`G4HV{GdL z{YBr8Q(00q>-!%V;zoUV`PzuqXO;|UvFV1?UPa}@CM8kQqTh;^ctoDJ+vNnTEUc}! z>*WN6Z!MH$0=POtDQ^@uhLSJoY$<4d+J#@wAKq^x*l<|6LswW6g$zhoTTQbp<+@wN z-*Nrk{rBSmg^d+A1Y?e#1F8yRT*3O{7!e`qIhgxLI#RsRJ=Y2|0y{YxeN-4RN&bTY z$!k(}MN>0!E2j}N$lgkr@U^v~!g{|VY}x7?&f)2P2^Ufg%S}>eNMn6k3dugQs~|G@ zmyayXyfoEO7i1i91cN7VYBFAtG;y4?_Tl#1;romAX#JGK4EvZ2`(K?9si5Fnm+Q6z zz{-9F936O+En6YC`=x0>#R}Lk@s+5R5Uy@1o8Ri3esr3%21M%3G%fBR)*|yN={GE) zg`!OysUdF_SYS+l(p^YT2lW(mJiQ|voTREL)atwDRp+Ty#dMMr(uOcfv2#&UhomI` z%dXd`pMNx|Sc?v)H9bH4Ztj6hg?94ax_-PL^0yUptjMI={JTm*(e5~NeCN+_gsv|5 z$D~!#oJdL*@;&sVMoaQ7R8Bh{C!fsxuNUL3UZXIKOr(ouKXb!GYd3HYw?Hp&{jHM8 z8Ea3v0`_CKxVB@8k54nr;VEvnNnjULQkkkf^3)$^&M%Tgu^WDsV?C_HnD;E8DSMcW zpe=k`6DLYlLUUpXZ@e*;mn<0)UqEx3Uo_=;{kJd%-JZs-YCp28L8n1oYK5Rn8T-#L zcz;D%RIdlb2bXfYl7QrAuk{DKQ%y(y$tN$2k9ai5{waob;26{Cn;vi9$HL#sd=1eIqb(gehZdZ9kY57$ALcD)o~f}4-~i;IdX0hAqHW}M1t z_GQ^<_Y6HISVXfLJmN)I61^-`j(CmMQ1Te6>`JPbovhq!)?@;6u4$!~KgBz7vB;yD zQpZOl&7X}k929^}BbGet?*Lv$+j{}$o-`N&GvM9ZhW8wq&`iTQ-wfZ(laP#@@#VHX z<9UIT4x962HnH-OHRf`HVbjfi#NNnaTF8BH-=GnroG}p7aOGw&>A^uaP`0mvB0-2A z=5E4>p2ehTS@P)b{l#rN3TYgJvZkFE(V+o7K21E!QTL%#gs;Lu&LB>49+Yi6%7Ih8 z-oE$>??voz75+VqRL(Yg7e<3VPY_U3Jt#ilYAP;;qtm8PQ~l*%?$5HJgf>gmXGAeB zdchG(kHm?bS?`+WpdWb^U*?NiDpx-2?gc0#TB41}y>dgt~#H(niCS z2?A6w3DqyzUo5A)fhLZdvtbpNR&Bs`c4}&~ZHHEe%@oz;F1?rE!|S&b5w(NvS&_a- z$;hT5#x&}>uTL|}Rw!Jaw?R$YY^Y0)eCp}+a^Q%J`R_~<$B*>#8YTREFF9)k(~psw z2j8J>*R(9FWhsU-^wH=ud)cK}kIM)I=k_CQX{Xa}A7p%&E>s=i=W>D%>tu@G-pbbZ z3c6HXs465A)GvB|3xrdb`S55>NvZ>vK&cDY6{GnxYd?KQOcrrCIw&PCpo5-tE_O7z z`q@BjFvRb zQl5dyBU9{*L3XcikNPtxKte;7+_3O+;~k7xmb7=?ZQi@+dhp8e!yTw`-iT>2CtqL2sHs4}cs6rmm(cr~$^G(|?6Va%_OKU7k`#GVwPUJ1% z^`gj<;j*6A&}-VSo+Zat=MU&hD_QSJ9kA2jP&WjXs97(*L7btj0R(xtpecmeKjr>F;RbCajz-^iN?C z?}CG5C`a*kY0PEa%^iPy3r3G04nfD(!?=o(BSyREczC{oaQNc-ZQ-w@_GS9QNREgR zEZ&0PA3w`Vwr(-1x|)P$j&G`n9$r|u`Ey2By12z;`UnhkbXO z>Ww9;7*{}Vex&QTpq&f;LhdSL%D<*pebh%pD9^qN!B~t~*9Z23BnZEP1k8X5*_1Q`2UM)o@5i&`g$RU5|if*D=SaHV+ zD?d{Hnj2?~tHG`E4>NZHFLuPym6mqWo^zB{2@JB&U(XkuE|E>=3A!f&?mHW9RyI2- zG~Z?O(;u33Z=KnHX5E2p_uH^JoUu()14sU86CC^e-=ClVn?um~18UQtZ~sMuYwW)~ Na#G5Y_2MR>{|}a~;Qjyr literal 0 HcmV?d00001 From a37ac093a67ac37dddba3d94a5e20ec688383417 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Thu, 2 Jan 2020 15:30:48 +0100 Subject: [PATCH 136/257] Fixed the compactor successfully exiting when actually an error occurred while compacting a blocks group (#1931) Signed-off-by: Marco Pracucci --- CHANGELOG.md | 1 + pkg/compact/compact.go | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7652cc555f..e6dc6f9a2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1856](https://github.com/thanos-io/thanos/pull/1856) Receive: close DBReadOnly after flushing to fix a memory leak. - [#1882](https://github.com/thanos-io/thanos/pull/1882) Receive: upload to object storage as 'receive' rather than 'sidecar'. - [#1907](https://github.com/thanos-io/thanos/pull/1907) Store: Fixed the duration unit for the metric `thanos_bucket_store_series_gate_duration_seconds`. +- [#1931](https://github.com/thanos-io/thanos/pull/1931) Compact: Fixed the compactor successfully exiting when actually an error occurred while compacting a blocks group. ### Added - [#1852](https://github.com/thanos-io/thanos/pull/1852) Add support for `AWS_CONTAINER_CREDENTIALS_FULL_URI` by upgrading to minio-go v6.0.44 diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index b3d1c32ac9..b853ff5be1 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -1131,10 +1131,13 @@ func (c *BucketCompactor) Compact(ctx context.Context) error { } // Send all groups found during this pass to the compaction workers. + var groupErrs terrors.MultiError + groupLoop: for _, g := range groups { select { - case err = <-errChan: + case groupErr := <-errChan: + groupErrs.Add(groupErr) break groupLoop case groupChan <- g: } @@ -1142,15 +1145,16 @@ func (c *BucketCompactor) Compact(ctx context.Context) error { close(groupChan) wg.Wait() + // Collect any other error reported by the workers, or any error reported + // while we were waiting for the last batch of groups to run the compaction. close(errChan) + for groupErr := range errChan { + groupErrs.Add(groupErr) + } + workCtxCancel() - if err != nil { - errs := terrors.MultiError{err} - // Collect any other errors reported by the workers. - for e := range errChan { - errs.Add(e) - } - return errs + if len(groupErrs) > 0 { + return groupErrs } if finishedAllGroups { From bb346c0296dab3fb1bdd4fee4f7695b241c4d835 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 3 Jan 2020 13:02:57 +0100 Subject: [PATCH 137/257] Add memcached support to index cache (#1881) * Moved index cache key struct outside of the file containing the in-memory cache backend because generic Signed-off-by: Marco Pracucci * Added cacheKey.string() to make it memcached friendly Signed-off-by: Marco Pracucci * Added MemcachedIndexCache support Signed-off-by: Marco Pracucci * Export Prometheus metrics from MemcachedIndexCache Signed-off-by: Marco Pracucci * Fixed linter issues Signed-off-by: Marco Pracucci * Simplified workers wait group in memcachedClient Signed-off-by: Marco Pracucci * Fixed memcached client GetMulti() results batches channel buffer Signed-off-by: Marco Pracucci * Wait for addrs resolution gorouting to complete on memcachedClient.Stop() Signed-off-by: Marco Pracucci * Return struct from NewMemcachedClient() Signed-off-by: Marco Pracucci * Update pkg/cacheutil/memcached_client.go Co-Authored-By: Bartlomiej Plotka Signed-off-by: Marco Pracucci * Simplified memcachedClient tests Signed-off-by: Marco Pracucci * Cleaned up code based on feedback Signed-off-by: Marco Pracucci * Removed error from GetMulti() return and introduced metrics and tracing in MemcachedClient Signed-off-by: Marco Pracucci * Fixed compilation errors in store E2E tests Signed-off-by: Marco Pracucci * Added leaktest check to all tests Signed-off-by: Marco Pracucci * Introduced --index.cache-config support Signed-off-by: Marco Pracucci * Fixed compilation errors in store E2E tests Signed-off-by: Marco Pracucci * Updated store flags doc Signed-off-by: Marco Pracucci * Updated index cache doc Signed-off-by: Marco Pracucci * Updated changelog Signed-off-by: Marco Pracucci * Increased default memcached client timeout from 100ms to 500ms Signed-off-by: Marco Pracucci * Disabled memcached client max batch size by default Signed-off-by: Marco Pracucci * Fixed index cache key max length Signed-off-by: Marco Pracucci * Removed TODO from memcached client since looks the general consensus is to have a global limit on the max concurrent batches Signed-off-by: Marco Pracucci * Allow to configure in-memory index cache using byte units Signed-off-by: Marco Pracucci * Refactored index cache config file doc Signed-off-by: Marco Pracucci * Fixed nits in comments Signed-off-by: Marco Pracucci * Replaced hardcoded 16 with ULID calculated length based on review comment Signed-off-by: Marco Pracucci * Do not expose jumpHash func Signed-off-by: Marco Pracucci * Updated changelog Signed-off-by: Marco Pracucci * Switched MemcachedClient GetMulti() concurrency limit to a gate Signed-off-by: Marco Pracucci * Fixed flaky tests Signed-off-by: Marco Pracucci * Fixed typos in comments Signed-off-by: Marco Pracucci * Renamed memcached config option addrs to addresses Signed-off-by: Marco Pracucci Co-authored-by: Bartlomiej Plotka --- CHANGELOG.md | 1 + cmd/thanos/store.go | 31 +- docs/components/store.md | 72 ++- go.mod | 6 +- go.sum | 4 + pkg/cacheutil/jump_hash.go | 18 + pkg/cacheutil/memcached_client.go | 441 ++++++++++++++++++ pkg/cacheutil/memcached_client_test.go | 400 ++++++++++++++++ pkg/cacheutil/memcached_server_selector.go | 98 ++++ .../memcached_server_selector_test.go | 184 ++++++++ pkg/{store => gate}/gate.go | 2 +- pkg/store/bucket.go | 13 +- pkg/store/bucket_e2e_test.go | 42 +- pkg/store/cache/cache.go | 69 ++- pkg/store/cache/cache_test.go | 99 ++++ pkg/store/cache/factory.go | 58 +++ pkg/store/cache/inmemory.go | 93 ++-- pkg/store/cache/inmemory_test.go | 111 +++-- pkg/store/cache/memcached.go | 177 +++++++ pkg/store/cache/memcached_test.go | 247 ++++++++++ pkg/store/cache/units.go | 29 ++ pkg/store/cache/units_test.go | 21 + scripts/cfggen/main.go | 21 +- 23 files changed, 2104 insertions(+), 133 deletions(-) create mode 100644 pkg/cacheutil/jump_hash.go create mode 100644 pkg/cacheutil/memcached_client.go create mode 100644 pkg/cacheutil/memcached_client_test.go create mode 100644 pkg/cacheutil/memcached_server_selector.go create mode 100644 pkg/cacheutil/memcached_server_selector_test.go rename pkg/{store => gate}/gate.go (99%) create mode 100644 pkg/store/cache/cache_test.go create mode 100644 pkg/store/cache/factory.go create mode 100644 pkg/store/cache/memcached.go create mode 100644 pkg/store/cache/memcached_test.go create mode 100644 pkg/store/cache/units.go create mode 100644 pkg/store/cache/units_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e6dc6f9a2e..cab5a36852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1854](https://github.com/thanos-io/thanos/pull/1854) Update Rule UI to support alerts count displaying and filtering. - [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add TLS and authentication support for Alertmanager with the `--alertmanagers.config` and `--alertmanagers.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. - [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add a new `--alertmanagers.sd-dns-interval` CLI option to specify the interval between DNS resolutions of Alertmanager hosts. +- [#1881](https://github.com/thanos-io/thanos/pull/1881) Store Gateway: memcached support for index cache. See [documentation](docs/components/store.md/#index-cache) for further information. ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 0a3af11677..1ec81d87b6 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -36,9 +36,13 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() - indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). + indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the in-memory index cache. Ignored if --index-cache.config or --index-cache.config-file option is specified."). Default("250MB").Bytes() + indexCacheConfig := extflag.RegisterPathOrContent(cmd, "index-cache.config", + "YAML file that contains index cache configuration. See format details: https://thanos.io/components/store.md/#index-cache", + false) + chunkPoolSize := cmd.Flag("chunk-pool-size", "Maximum size of concurrently allocatable bytes for chunks."). Default("2GB").Bytes() @@ -77,6 +81,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { logger, reg, tracer, + indexCacheConfig, objStoreConfig, *dataDir, *grpcBindAddr, @@ -110,6 +115,7 @@ func runStore( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, + indexCacheConfig *extflag.PathOrContent, objStoreConfig *extflag.PathOrContent, dataDir string, grpcBindAddr string, @@ -169,6 +175,11 @@ func runStore( return err } + indexCacheContentYaml, err := indexCacheConfig.Content() + if err != nil { + return errors.Wrap(err, "get content of index cache configuration") + } + // Ensure we close up everything properly. defer func() { if err != nil { @@ -176,13 +187,17 @@ func runStore( } }() - // TODO(bwplotka): Add as a flag? - maxItemSizeBytes := indexCacheSizeBytes / 2 - - indexCache, err := storecache.NewInMemoryIndexCache(logger, reg, storecache.Opts{ - MaxSizeBytes: indexCacheSizeBytes, - MaxItemSizeBytes: maxItemSizeBytes, - }) + // Create the index cache loading its config from config file, while keeping + // backward compatibility with the pre-config file era. + var indexCache storecache.IndexCache + if len(indexCacheContentYaml) > 0 { + indexCache, err = storecache.NewIndexCache(logger, indexCacheContentYaml, reg) + } else { + indexCache, err = storecache.NewInMemoryIndexCacheWithConfig(logger, reg, storecache.InMemoryIndexCacheConfig{ + MaxSize: storecache.Bytes(indexCacheSizeBytes), + MaxItemSize: storecache.DefaultInMemoryIndexCacheConfig.MaxItemSize, + }) + } if err != nil { return errors.Wrap(err, "create index cache") } diff --git a/docs/components/store.md b/docs/components/store.md index 3b2a7f5737..c25512993c 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -68,7 +68,19 @@ Flags: CA is specified, there is no client verification on server side. (tls.NoClientCert) --data-dir="./data" Data directory in which to cache remote blocks. - --index-cache-size=250MB Maximum size of items held in the index cache. + --index-cache-size=250MB Maximum size of items held in the in-memory + index cache. Ignored if --index-cache.config or + --index-cache.config-file option is specified. + --index-cache.config-file= + Path to YAML file that contains index cache + configuration. See format details: + https://thanos.io/components/store.md/#index-cache + --index-cache.config= + Alternative to 'index-cache.config-file' flag + (lower priority). Content of YAML file that + contains index cache configuration. See format + details: + https://thanos.io/components/store.md/#index-cache --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes for chunks. --store.grpc.series-sample-limit=0 @@ -151,3 +163,61 @@ Filtering is done on a Chunk level, so Thanos Store might still return Samples w - `/-/ready` starts after all the bootstrapping completed (e.g initial index building) and ready to serve traffic. > NOTE: Metric endpoint starts immediately so, make sure you set up readiness probe on designated HTTP `/-/ready` path. + +## Index cache + +Thanos Store Gateway supports an index cache to speed up postings and series lookups from TSDB blocks indexes. Two types of caches are supported: + +- `in-memory` (_default_) +- `memcached` + +### In-memory index cache + +The `in-memory` index cache is enabled by default and its max size can be configured through the flag `--index-cache-size`. + +Alternatively, the `in-memory` index cache can also by configured using `--index-cache.config-file` to reference to the configuration file or `--index-cache.config` to put yaml config directly: + +[embedmd]:# (../flags/config_index_cache_in_memory.txt yaml) +```yaml +type: IN-MEMORY +config: + max_size: 0 + max_item_size: 0 +``` + +All the settings are **optional**: + +- `max_size`: overall maximum number of bytes cache can contain. The value should be specified with a bytes unit (ie. `250MB`). +- `max_item_size`: maximum size of single item, in bytes. The value should be specified with a bytes unit (ie. `125MB`). + +### Memcached index cache + +The `memcached` index cache allows to use [Memcached](https://memcached.org) as cache backend. This cache type is configured using `--index-cache.config-file` to reference to the configuration file or `--index-cache.config` to put yaml config directly: + +[embedmd]:# (../flags/config_index_cache_memcached.txt yaml) +```yaml +type: MEMCACHED +config: + addresses: [] + timeout: 0s + max_idle_connections: 0 + max_async_concurrency: 0 + max_async_buffer_size: 0 + max_get_multi_concurrency: 0 + max_get_multi_batch_size: 0 + dns_provider_update_interval: 0s +``` + +The **required** settings are: + +- `addresses`: list of memcached addresses, that will get resolved with the [DNS service discovery](../service-discovery.md/#dns-service-discovery) provider. + +While the remaining settings are **optional**: + +- `timeout`: the socket read/write timeout. +- `max_idle_connections`: maximum number of idle connections that will be maintained per address. +- `max_async_concurrency`: maximum number of concurrent asynchronous operations can occur. +- `max_async_buffer_size`: maximum number of enqueued asynchronous operations allowed. +- `max_get_multi_concurrency`: maximum number of concurrent connections when fetching keys. If set to `0`, the concurrency is unlimited. +- `max_get_multi_batch_size`: maximum number of keys a single underlying operation should fetch. If more keys are specified, internally keys are splitted into multiple batches and fetched concurrently, honoring `max_get_multi_concurrency`. If set to `0`, the batch size is unlimited. +- `dns_provider_update_interval`: the DNS discovery update interval. diff --git a/go.mod b/go.mod index 171dd7188a..bae87de512 100644 --- a/go.mod +++ b/go.mod @@ -13,17 +13,19 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.2.1-0.20191028180845-3492b2aff503 // indirect github.com/NYTimes/gziphandler v1.1.1 github.com/OneOfOne/xxhash v1.2.6 // indirect - github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible github.com/armon/go-metrics v0.3.0 github.com/aws/aws-sdk-go v1.25.35 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect + github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/elastic/go-sysinfo v1.1.1 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/evanphx/json-patch v4.5.0+incompatible // indirect + github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb github.com/fatih/structtag v1.1.0 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 @@ -83,7 +85,7 @@ require ( go.opencensus.io v0.22.2 // indirect go.uber.org/atomic v1.5.0 // indirect go.uber.org/automaxprocs v1.2.0 - golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 // indirect + golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 // indirect diff --git a/go.sum b/go.sum index 42cbdc4313..724168b406 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= +github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v0.0.0-20181017004759-096ff4a8a059/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -134,6 +136,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM= +github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structtag v1.1.0 h1:6j4mUV/ES2duvnAzKMFkN6/A5mCaNYPD3xfbAkLLOF8= github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= diff --git a/pkg/cacheutil/jump_hash.go b/pkg/cacheutil/jump_hash.go new file mode 100644 index 0000000000..4b684b7384 --- /dev/null +++ b/pkg/cacheutil/jump_hash.go @@ -0,0 +1,18 @@ +package cacheutil + +// jumpHash consistently chooses a hash bucket number in the range +// [0, numBuckets) for the given key. numBuckets must be >= 1. +// +// Copied from github.com/dgryski/go-jump/blob/master/jump.go (MIT license). +func jumpHash(key uint64, numBuckets int) int32 { + var b int64 = -1 + var j int64 + + for j < int64(numBuckets) { + b = j + key = key*2862933555777941757 + 1 + j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1))) + } + + return int32(b) +} diff --git a/pkg/cacheutil/memcached_client.go b/pkg/cacheutil/memcached_client.go new file mode 100644 index 0000000000..91d4552dd1 --- /dev/null +++ b/pkg/cacheutil/memcached_client.go @@ -0,0 +1,441 @@ +package cacheutil + +import ( + "context" + "sync" + "time" + + "github.com/bradfitz/gomemcache/memcache" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/thanos-io/thanos/pkg/discovery/dns" + "github.com/thanos-io/thanos/pkg/extprom" + "github.com/thanos-io/thanos/pkg/gate" + "github.com/thanos-io/thanos/pkg/tracing" + yaml "gopkg.in/yaml.v2" +) + +const ( + opSet = "set" + opGetMulti = "getmulti" +) + +var ( + errMemcachedAsyncBufferFull = errors.New("the async buffer is full") + errMemcachedConfigNoAddrs = errors.New("no memcached addresses provided") + + defaultMemcachedClientConfig = MemcachedClientConfig{ + Timeout: 500 * time.Millisecond, + MaxIdleConnections: 100, + MaxAsyncConcurrency: 20, + MaxAsyncBufferSize: 10000, + MaxGetMultiConcurrency: 100, + MaxGetMultiBatchSize: 0, + DNSProviderUpdateInterval: 10 * time.Second, + } +) + +// MemcachedClient is a high level client to interact with memcached. +type MemcachedClient interface { + // GetMulti fetches multiple keys at once from memcached. In case of error, + // an empty map is returned and the error tracked/logged. + GetMulti(ctx context.Context, keys []string) map[string][]byte + + // SetAsync enqueues an asynchronous operation to store a key into memcached. + // Returns an error in case it fails to enqueue the operation. In case the + // underlying async operation will fail, the error will be tracked/logged. + SetAsync(ctx context.Context, key string, value []byte, ttl time.Duration) error + + // Stop client and release underlying resources. + Stop() +} + +// memcachedClientBackend is an interface used to mock the underlying client in tests. +type memcachedClientBackend interface { + GetMulti(keys []string) (map[string]*memcache.Item, error) + Set(item *memcache.Item) error +} + +// MemcachedClientConfig is the config accepted by MemcachedClient. +type MemcachedClientConfig struct { + // Addresses specifies the list of memcached addresses. The addresses get + // resolved with the DNS provider. + Addresses []string `yaml:"addresses"` + + // Timeout specifies the socket read/write timeout. + Timeout time.Duration `yaml:"timeout"` + + // MaxIdleConnections specifies the maximum number of idle connections that + // will be maintained per address. For better performances, this should be + // set to a number higher than your peak parallel requests. + MaxIdleConnections int `yaml:"max_idle_connections"` + + // MaxAsyncConcurrency specifies the maximum number of concurrent asynchronous + // operations can occur. + MaxAsyncConcurrency int `yaml:"max_async_concurrency"` + + // MaxAsyncBufferSize specifies the maximum number of enqueued asynchronous + // operations allowed. + MaxAsyncBufferSize int `yaml:"max_async_buffer_size"` + + // MaxGetMultiConcurrency specifies the maximum number of concurrent connections + // running GetMulti() operations. If set to 0, concurrency is unlimited. + MaxGetMultiConcurrency int `yaml:"max_get_multi_concurrency"` + + // MaxGetMultiBatchSize specifies the maximum number of keys a single underlying + // GetMulti() should run. If more keys are specified, internally keys are splitted + // into multiple batches and fetched concurrently, honoring MaxGetMultiConcurrency + // parallelism. If set to 0, the max batch size is unlimited. + MaxGetMultiBatchSize int `yaml:"max_get_multi_batch_size"` + + // DNSProviderUpdateInterval specifies the DNS discovery update interval. + DNSProviderUpdateInterval time.Duration `yaml:"dns_provider_update_interval"` +} + +func (c *MemcachedClientConfig) validate() error { + if len(c.Addresses) == 0 { + return errMemcachedConfigNoAddrs + } + + return nil +} + +// parseMemcachedClientConfig unmarshals a buffer into a MemcachedClientConfig with default values. +func parseMemcachedClientConfig(conf []byte) (MemcachedClientConfig, error) { + config := defaultMemcachedClientConfig + if err := yaml.Unmarshal(conf, &config); err != nil { + return MemcachedClientConfig{}, err + } + + return config, nil +} + +type memcachedClient struct { + logger log.Logger + config MemcachedClientConfig + client memcachedClientBackend + selector *MemcachedJumpHashSelector + + // DNS provider used to keep the memcached servers list updated. + dnsProvider *dns.Provider + + // Channel used to notify internal goroutines when they should quit. + stop chan struct{} + + // Channel used to enqueue async operations. + asyncQueue chan func() + + // Gate used to enforce the max number of concurrent GetMulti() operations. + getMultiGate *gate.Gate + + // Wait group used to wait all workers on stopping. + workers sync.WaitGroup + + // Tracked metrics. + operations *prometheus.CounterVec + failures *prometheus.CounterVec + duration *prometheus.HistogramVec +} + +type memcachedGetMultiResult struct { + items map[string]*memcache.Item + err error +} + +// NewMemcachedClient makes a new MemcachedClient. +func NewMemcachedClient(logger log.Logger, name string, conf []byte, reg prometheus.Registerer) (*memcachedClient, error) { + config, err := parseMemcachedClientConfig(conf) + if err != nil { + return nil, err + } + + return NewMemcachedClientWithConfig(logger, name, config, reg) +} + +// NewMemcachedClientWithConfig makes a new MemcachedClient. +func NewMemcachedClientWithConfig(logger log.Logger, name string, config MemcachedClientConfig, reg prometheus.Registerer) (*memcachedClient, error) { + if err := config.validate(); err != nil { + return nil, err + } + + // We use a custom servers selector in order to use a jump hash + // for servers selection. + selector := &MemcachedJumpHashSelector{} + + client := memcache.NewFromSelector(selector) + client.Timeout = config.Timeout + client.MaxIdleConns = config.MaxIdleConnections + + return newMemcachedClient(logger, name, client, selector, config, reg) +} + +func newMemcachedClient( + logger log.Logger, + name string, + client memcachedClientBackend, + selector *MemcachedJumpHashSelector, + config MemcachedClientConfig, + reg prometheus.Registerer, +) (*memcachedClient, error) { + dnsProvider := dns.NewProvider( + logger, + extprom.WrapRegistererWithPrefix("thanos_memcached_", reg), + dns.ResolverType(dns.GolangResolverType), + ) + + c := &memcachedClient{ + logger: logger, + config: config, + client: client, + selector: selector, + dnsProvider: dnsProvider, + asyncQueue: make(chan func(), config.MaxAsyncBufferSize), + stop: make(chan struct{}, 1), + getMultiGate: gate.NewGate( + config.MaxGetMultiConcurrency, + extprom.WrapRegistererWithPrefix("thanos_memcached_getmulti_", reg), + ), + } + + c.operations = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_memcached_operations_total", + Help: "Total number of operations against memcached.", + ConstLabels: prometheus.Labels{"name": name}, + }, []string{"operation"}) + + c.failures = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_memcached_operation_failures_total", + Help: "Total number of operations against memcached that failed.", + ConstLabels: prometheus.Labels{"name": name}, + }, []string{"operation"}) + + c.duration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "thanos_memcached_operation_duration_seconds", + Help: "Duration of operations against memcached.", + ConstLabels: prometheus.Labels{"name": name}, + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1}, + }, []string{"operation"}) + + if reg != nil { + reg.MustRegister(c.operations, c.failures, c.duration) + } + + // As soon as the client is created it must ensure that memcached server + // addresses are resolved, so we're going to trigger an initial addresses + // resolution here. + if err := c.resolveAddrs(); err != nil { + return nil, err + } + + c.workers.Add(1) + go c.resolveAddrsLoop() + + // Start a number of goroutines - processing async operations - equal + // to the max concurrency we have. + c.workers.Add(c.config.MaxAsyncConcurrency) + for i := 0; i < c.config.MaxAsyncConcurrency; i++ { + go c.asyncQueueProcessLoop() + } + + return c, nil +} + +func (c *memcachedClient) Stop() { + close(c.stop) + + // Wait until all workers have terminated. + c.workers.Wait() +} + +func (c *memcachedClient) SetAsync(ctx context.Context, key string, value []byte, ttl time.Duration) error { + return c.enqueueAsync(func() { + start := time.Now() + c.operations.WithLabelValues(opSet).Inc() + + span, _ := tracing.StartSpan(ctx, "memcached_set") + err := c.client.Set(&memcache.Item{ + Key: key, + Value: value, + Expiration: int32(time.Now().Add(ttl).Unix()), + }) + span.Finish() + if err != nil { + c.failures.WithLabelValues(opSet).Inc() + level.Warn(c.logger).Log("msg", "failed to store item to memcached", "key", key, "err", err) + return + } + + c.duration.WithLabelValues(opSet).Observe(time.Since(start).Seconds()) + }) +} + +func (c *memcachedClient) GetMulti(ctx context.Context, keys []string) map[string][]byte { + batches, err := c.getMultiBatched(ctx, keys) + if err != nil { + level.Warn(c.logger).Log("msg", "failed to fetch items from memcached", "err", err) + + // In case we have both results and an error, it means some batch requests + // failed and other succeeded. In this case we prefer to log it and move on, + // given returning some results from the cache is better than returning + // nothing. + if len(batches) == 0 { + return nil + } + } + + hits := map[string][]byte{} + for _, items := range batches { + for key, item := range items { + hits[key] = item.Value + } + } + + return hits +} + +func (c *memcachedClient) getMultiBatched(ctx context.Context, keys []string) ([]map[string]*memcache.Item, error) { + // Do not batch if the input keys are less than the max batch size. + if (c.config.MaxGetMultiBatchSize <= 0) || (len(keys) <= c.config.MaxGetMultiBatchSize) { + items, err := c.getMultiSingle(ctx, keys) + if err != nil { + return nil, err + } + + return []map[string]*memcache.Item{items}, nil + } + + // Calculate the number of expected results. + batchSize := c.config.MaxGetMultiBatchSize + numResults := len(keys) / batchSize + if len(keys)%batchSize != 0 { + numResults++ + } + + // Spawn a goroutine for each batch request. The max concurrency will be + // enforced by getMultiSingle(). + results := make(chan *memcachedGetMultiResult, numResults) + defer close(results) + + for batchStart := 0; batchStart < len(keys); batchStart += batchSize { + batchEnd := batchStart + batchSize + if batchEnd > len(keys) { + batchEnd = len(keys) + } + + batchKeys := keys[batchStart:batchEnd] + + c.workers.Add(1) + go func() { + defer c.workers.Done() + + res := &memcachedGetMultiResult{} + res.items, res.err = c.getMultiSingle(ctx, batchKeys) + + results <- res + }() + } + + // Wait for all batch results. In case of error, we keep + // track of the last error occurred. + items := make([]map[string]*memcache.Item, 0, numResults) + var lastErr error + + for i := 0; i < numResults; i++ { + result := <-results + if result.err != nil { + lastErr = result.err + continue + } + + items = append(items, result.items) + } + + return items, lastErr +} + +func (c *memcachedClient) getMultiSingle(ctx context.Context, keys []string) (map[string]*memcache.Item, error) { + // Wait until we get a free slot from the gate, if the max + // concurrency should be enforced. + if c.config.MaxGetMultiConcurrency > 0 { + span, _ := tracing.StartSpan(ctx, "memcached_getmulti_gate_ismyturn") + err := c.getMultiGate.IsMyTurn(ctx) + span.Finish() + if err != nil { + return nil, errors.Wrapf(err, "failed to wait for turn") + } + defer c.getMultiGate.Done() + } + + start := time.Now() + c.operations.WithLabelValues(opGetMulti).Inc() + + span, _ := tracing.StartSpan(ctx, "memcached_getmulti") + items, err := c.client.GetMulti(keys) + span.Finish() + if err != nil { + c.failures.WithLabelValues(opGetMulti).Inc() + } else { + c.duration.WithLabelValues(opGetMulti).Observe(time.Since(start).Seconds()) + } + + return items, err +} + +func (c *memcachedClient) enqueueAsync(op func()) error { + select { + case c.asyncQueue <- op: + return nil + default: + return errMemcachedAsyncBufferFull + } +} + +func (c *memcachedClient) asyncQueueProcessLoop() { + defer c.workers.Done() + + for { + select { + case op := <-c.asyncQueue: + op() + case <-c.stop: + return + } + } +} + +func (c *memcachedClient) resolveAddrsLoop() { + defer c.workers.Done() + + ticker := time.NewTicker(c.config.DNSProviderUpdateInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + err := c.resolveAddrs() + if err != nil { + level.Warn(c.logger).Log("msg", "failed update memcached servers list", "err", err) + } + case <-c.stop: + return + } + } +} + +func (c *memcachedClient) resolveAddrs() error { + // Resolve configured addresses with a reasonable timeout. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + c.dnsProvider.Resolve(ctx, c.config.Addresses) + + // Fail in case no server address is resolved. + servers := c.dnsProvider.Addresses() + if len(servers) == 0 { + return errors.New("no server address resolved") + } + + return c.selector.SetServers(servers...) +} diff --git a/pkg/cacheutil/memcached_client_test.go b/pkg/cacheutil/memcached_client_test.go new file mode 100644 index 0000000000..a1f77fe282 --- /dev/null +++ b/pkg/cacheutil/memcached_client_test.go @@ -0,0 +1,400 @@ +package cacheutil + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/bradfitz/gomemcache/memcache" + "github.com/fortytw2/leaktest" + "github.com/go-kit/kit/log" + prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestMemcachedClientConfig_validate(t *testing.T) { + tests := map[string]struct { + config MemcachedClientConfig + expected error + }{ + "should pass on valid config": { + config: MemcachedClientConfig{ + Addresses: []string{"127.0.0.1:11211"}, + }, + expected: nil, + }, + "should fail on no addresses": { + config: MemcachedClientConfig{ + Addresses: []string{}, + }, + expected: errMemcachedConfigNoAddrs, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + testutil.Equals(t, testData.expected, testData.config.validate()) + }) + } +} + +func TestNewMemcachedClient(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + // Should return error on empty YAML config. + conf := []byte{} + cache, err := NewMemcachedClient(log.NewNopLogger(), "test", conf, nil) + testutil.NotOk(t, err) + testutil.Equals(t, (*memcachedClient)(nil), cache) + + // Should return error on invalid YAML config. + conf = []byte("invalid") + cache, err = NewMemcachedClient(log.NewNopLogger(), "test", conf, nil) + testutil.NotOk(t, err) + testutil.Equals(t, (*memcachedClient)(nil), cache) + + // Should instance a memcached client with minimum YAML config. + conf = []byte(` +addresses: + - 127.0.0.1:11211 + - 127.0.0.2:11211 +`) + cache, err = NewMemcachedClient(log.NewNopLogger(), "test", conf, nil) + testutil.Ok(t, err) + defer cache.Stop() + + testutil.Equals(t, []string{"127.0.0.1:11211", "127.0.0.2:11211"}, cache.config.Addresses) + testutil.Equals(t, defaultMemcachedClientConfig.Timeout, cache.config.Timeout) + testutil.Equals(t, defaultMemcachedClientConfig.MaxIdleConnections, cache.config.MaxIdleConnections) + testutil.Equals(t, defaultMemcachedClientConfig.MaxAsyncConcurrency, cache.config.MaxAsyncConcurrency) + testutil.Equals(t, defaultMemcachedClientConfig.MaxAsyncBufferSize, cache.config.MaxAsyncBufferSize) + testutil.Equals(t, defaultMemcachedClientConfig.DNSProviderUpdateInterval, cache.config.DNSProviderUpdateInterval) + testutil.Equals(t, defaultMemcachedClientConfig.MaxGetMultiConcurrency, cache.config.MaxGetMultiConcurrency) + testutil.Equals(t, defaultMemcachedClientConfig.MaxGetMultiBatchSize, cache.config.MaxGetMultiBatchSize) + + // Should instance a memcached client with configured YAML config. + conf = []byte(` +addresses: + - 127.0.0.1:11211 + - 127.0.0.2:11211 +timeout: 1s +max_idle_connections: 1 +max_async_concurrency: 1 +max_async_buffer_size: 1 +max_get_multi_concurrency: 1 +max_get_multi_batch_size: 1 +dns_provider_update_interval: 1s +`) + cache, err = NewMemcachedClient(log.NewNopLogger(), "test", conf, nil) + testutil.Ok(t, err) + defer cache.Stop() + + testutil.Equals(t, []string{"127.0.0.1:11211", "127.0.0.2:11211"}, cache.config.Addresses) + testutil.Equals(t, 1*time.Second, cache.config.Timeout) + testutil.Equals(t, 1, cache.config.MaxIdleConnections) + testutil.Equals(t, 1, cache.config.MaxAsyncConcurrency) + testutil.Equals(t, 1, cache.config.MaxAsyncBufferSize) + testutil.Equals(t, 1*time.Second, cache.config.DNSProviderUpdateInterval) + testutil.Equals(t, 1, cache.config.MaxGetMultiConcurrency) + testutil.Equals(t, 1, cache.config.MaxGetMultiBatchSize) +} + +func TestMemcachedClient_SetAsync(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + ctx := context.Background() + config := defaultMemcachedClientConfig + config.Addresses = []string{"127.0.0.1:11211"} + backendMock := newMemcachedClientBackendMock() + + client, err := prepare(config, backendMock) + testutil.Ok(t, err) + defer client.Stop() + + testutil.Ok(t, client.SetAsync(ctx, "key-1", []byte("value-1"), time.Second)) + testutil.Ok(t, client.SetAsync(ctx, "key-2", []byte("value-2"), time.Second)) + testutil.Ok(t, backendMock.waitItems(2)) + + testutil.Equals(t, 2.0, prom_testutil.ToFloat64(client.operations.WithLabelValues(opSet))) + testutil.Equals(t, 0.0, prom_testutil.ToFloat64(client.operations.WithLabelValues(opGetMulti))) + testutil.Equals(t, 0.0, prom_testutil.ToFloat64(client.failures.WithLabelValues(opSet))) +} + +func TestMemcachedClient_GetMulti(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + tests := map[string]struct { + maxBatchSize int + maxConcurrency int + mockedGetMultiErrors int + initialItems []memcache.Item + getKeys []string + expectedHits map[string][]byte + expectedGetMultiCount int + }{ + "should fetch keys in a single batch if the input keys is <= the max batch size": { + maxBatchSize: 2, + maxConcurrency: 5, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + }, + getKeys: []string{"key-1", "key-2"}, + expectedHits: map[string][]byte{ + "key-1": []byte("value-1"), + "key-2": []byte("value-2"), + }, + expectedGetMultiCount: 1, + }, + "should fetch keys in multiple batches if the input keys is > the max batch size": { + maxBatchSize: 2, + maxConcurrency: 5, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + {Key: "key-3", Value: []byte("value-3")}, + }, + getKeys: []string{"key-1", "key-2", "key-3"}, + expectedHits: map[string][]byte{ + "key-1": []byte("value-1"), + "key-2": []byte("value-2"), + "key-3": []byte("value-3"), + }, + expectedGetMultiCount: 2, + }, + "should fetch keys in multiple batches on input keys exact multiple of batch size": { + maxBatchSize: 2, + maxConcurrency: 5, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + {Key: "key-3", Value: []byte("value-3")}, + {Key: "key-4", Value: []byte("value-4")}, + }, + getKeys: []string{"key-1", "key-2", "key-3", "key-4"}, + expectedHits: map[string][]byte{ + "key-1": []byte("value-1"), + "key-2": []byte("value-2"), + "key-3": []byte("value-3"), + "key-4": []byte("value-4"), + }, + expectedGetMultiCount: 2, + }, + "should fetch keys in multiple batches on input keys exact multiple of batch size with max concurrency disabled (0)": { + maxBatchSize: 2, + maxConcurrency: 0, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + {Key: "key-3", Value: []byte("value-3")}, + {Key: "key-4", Value: []byte("value-4")}, + }, + getKeys: []string{"key-1", "key-2", "key-3", "key-4"}, + expectedHits: map[string][]byte{ + "key-1": []byte("value-1"), + "key-2": []byte("value-2"), + "key-3": []byte("value-3"), + "key-4": []byte("value-4"), + }, + expectedGetMultiCount: 2, + }, + "should fetch keys in multiple batches on input keys exact multiple of batch size with max concurrency lower than the batches": { + maxBatchSize: 1, + maxConcurrency: 1, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + {Key: "key-3", Value: []byte("value-3")}, + {Key: "key-4", Value: []byte("value-4")}, + }, + getKeys: []string{"key-1", "key-2", "key-3", "key-4"}, + expectedHits: map[string][]byte{ + "key-1": []byte("value-1"), + "key-2": []byte("value-2"), + "key-3": []byte("value-3"), + "key-4": []byte("value-4"), + }, + expectedGetMultiCount: 4, + }, + "should fetch keys in a single batch if max batch size is disabled (0)": { + maxBatchSize: 0, + maxConcurrency: 5, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + {Key: "key-3", Value: []byte("value-3")}, + {Key: "key-4", Value: []byte("value-4")}, + }, + getKeys: []string{"key-1", "key-2", "key-3", "key-4"}, + expectedHits: map[string][]byte{ + "key-1": []byte("value-1"), + "key-2": []byte("value-2"), + "key-3": []byte("value-3"), + "key-4": []byte("value-4"), + }, + expectedGetMultiCount: 1, + }, + "should fetch keys in a single batch if max batch size is disabled (0) and max concurrency is disabled (0)": { + maxBatchSize: 0, + maxConcurrency: 0, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + {Key: "key-3", Value: []byte("value-3")}, + {Key: "key-4", Value: []byte("value-4")}, + }, + getKeys: []string{"key-1", "key-2", "key-3", "key-4"}, + expectedHits: map[string][]byte{ + "key-1": []byte("value-1"), + "key-2": []byte("value-2"), + "key-3": []byte("value-3"), + "key-4": []byte("value-4"), + }, + expectedGetMultiCount: 1, + }, + "should return no hits on all keys missing": { + maxBatchSize: 2, + maxConcurrency: 5, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + }, + getKeys: []string{"key-1", "key-2", "key-3", "key-4"}, + expectedHits: map[string][]byte{ + "key-1": []byte("value-1"), + "key-2": []byte("value-2"), + }, + expectedGetMultiCount: 2, + }, + "should return no hits on partial errors while fetching batches and no items found": { + maxBatchSize: 2, + maxConcurrency: 5, + mockedGetMultiErrors: 1, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + {Key: "key-3", Value: []byte("value-3")}, + }, + getKeys: []string{"key-5", "key-6", "key-7"}, + expectedHits: map[string][]byte{}, + expectedGetMultiCount: 2, + }, + "should return no hits on all errors while fetching batches": { + maxBatchSize: 2, + maxConcurrency: 5, + mockedGetMultiErrors: 2, + initialItems: []memcache.Item{ + {Key: "key-1", Value: []byte("value-1")}, + {Key: "key-2", Value: []byte("value-2")}, + {Key: "key-3", Value: []byte("value-3")}, + }, + getKeys: []string{"key-5", "key-6", "key-7"}, + expectedHits: nil, + expectedGetMultiCount: 2, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + ctx := context.Background() + config := defaultMemcachedClientConfig + config.Addresses = []string{"127.0.0.1:11211"} + config.MaxGetMultiBatchSize = testData.maxBatchSize + config.MaxGetMultiConcurrency = testData.maxConcurrency + + backendMock := newMemcachedClientBackendMock() + backendMock.getMultiErrors = testData.mockedGetMultiErrors + + client, err := prepare(config, backendMock) + testutil.Ok(t, err) + defer client.Stop() + + // Populate memcached with the initial items. + for _, item := range testData.initialItems { + testutil.Ok(t, client.SetAsync(ctx, item.Key, item.Value, time.Second)) + } + + // Wait until initial items have been added. + testutil.Ok(t, backendMock.waitItems(len(testData.initialItems))) + + // Read back the items. + testutil.Equals(t, testData.expectedHits, client.GetMulti(ctx, testData.getKeys)) + + // Ensure the client has interacted with the backend as expected. + backendMock.lock.Lock() + defer backendMock.lock.Unlock() + testutil.Equals(t, testData.expectedGetMultiCount, backendMock.getMultiCount) + + // Ensure metrics are tracked. + testutil.Equals(t, float64(testData.expectedGetMultiCount), prom_testutil.ToFloat64(client.operations.WithLabelValues(opGetMulti))) + testutil.Equals(t, float64(testData.mockedGetMultiErrors), prom_testutil.ToFloat64(client.failures.WithLabelValues(opGetMulti))) + }) + } +} + +func prepare(config MemcachedClientConfig, backendMock *memcachedClientBackendMock) (*memcachedClient, error) { + logger := log.NewNopLogger() + selector := &MemcachedJumpHashSelector{} + client, err := newMemcachedClient(logger, "test", backendMock, selector, config, nil) + + return client, err +} + +type memcachedClientBackendMock struct { + lock sync.Mutex + items map[string]*memcache.Item + getMultiCount int + getMultiErrors int +} + +func newMemcachedClientBackendMock() *memcachedClientBackendMock { + return &memcachedClientBackendMock{ + items: map[string]*memcache.Item{}, + } +} + +func (c *memcachedClientBackendMock) GetMulti(keys []string) (map[string]*memcache.Item, error) { + c.lock.Lock() + defer c.lock.Unlock() + + c.getMultiCount++ + if c.getMultiCount <= c.getMultiErrors { + return nil, errors.New("mocked GetMulti error") + } + + items := make(map[string]*memcache.Item) + for _, key := range keys { + if item, ok := c.items[key]; ok { + items[key] = item + } + } + + return items, nil +} + +func (c *memcachedClientBackendMock) Set(item *memcache.Item) error { + c.lock.Lock() + defer c.lock.Unlock() + + c.items[item.Key] = item + + return nil +} + +func (c *memcachedClientBackendMock) waitItems(expected int) error { + deadline := time.Now().Add(1 * time.Second) + + for time.Now().Before(deadline) { + c.lock.Lock() + count := len(c.items) + c.lock.Unlock() + + if count >= expected { + return nil + } + } + + return errors.New("timeout expired while waiting for items in the memcached mock") +} diff --git a/pkg/cacheutil/memcached_server_selector.go b/pkg/cacheutil/memcached_server_selector.go new file mode 100644 index 0000000000..1318574045 --- /dev/null +++ b/pkg/cacheutil/memcached_server_selector.go @@ -0,0 +1,98 @@ +package cacheutil + +import ( + "net" + "sync" + + "github.com/bradfitz/gomemcache/memcache" + "github.com/cespare/xxhash" + "github.com/facette/natsort" +) + +var ( + addrsPool = sync.Pool{ + New: func() interface{} { + addrs := make([]net.Addr, 0, 64) + return &addrs + }, + } +) + +// MemcachedJumpHashSelector implements the memcache.ServerSelector +// interface, utilizing a jump hash to distribute keys to servers. +// +// While adding or removing servers only requires 1/N keys to move, +// servers are treated as a stack and can only be pushed/popped. +// Therefore, MemcachedJumpHashSelector works best for servers +// with consistent DNS names where the naturally sorted order +// is predictable (ie. Kubernetes statefulsets). +type MemcachedJumpHashSelector struct { + // To avoid copy and pasting all memcache server list logic, + // we embed it and implement our features on top of it. + servers memcache.ServerList +} + +// SetServers changes a MemcachedJumpHashSelector's set of servers at +// runtime and is safe for concurrent use by multiple goroutines. +// +// Each server is given equal weight. A server is given more weight +// if it's listed multiple times. +// +// SetServers returns an error if any of the server names fail to +// resolve. No attempt is made to connect to the server. If any +// error occurs, no changes are made to the internal server list. +// +// To minimize the number of rehashes for keys when scaling the +// number of servers in subsequent calls to SetServers, servers +// are stored in natural sort order. +func (s *MemcachedJumpHashSelector) SetServers(servers ...string) error { + sortedServers := make([]string, len(servers)) + copy(sortedServers, servers) + natsort.Sort(sortedServers) + + return s.servers.SetServers(sortedServers...) +} + +// PickServer returns the server address that a given item +// should be shared onto. +func (s *MemcachedJumpHashSelector) PickServer(key string) (net.Addr, error) { + // Unfortunately we can't read the list of server addresses from + // the original implementation, so we use Each() to fetch all of them. + addrs := *(addrsPool.Get().(*[]net.Addr)) + err := s.servers.Each(func(addr net.Addr) error { + addrs = append(addrs, addr) + return nil + }) + if err != nil { + return nil, err + } + + // No need of a jump hash in case of 0 or 1 servers. + if len(addrs) == 0 { + addrs = (addrs)[:0] + addrsPool.Put(&addrs) + return nil, memcache.ErrNoServers + } + if len(addrs) == 1 { + addrs = (addrs)[:0] + addrsPool.Put(&addrs) + return (addrs)[0], nil + } + + // Pick a server using the jump hash. + cs := xxhash.Sum64String(key) + idx := jumpHash(cs, len(addrs)) + picked := (addrs)[idx] + + addrs = (addrs)[:0] + addrsPool.Put(&addrs) + + return picked, nil +} + +// Each iterates over each server and calls the given function. +// If f returns a non-nil error, iteration will stop and that +// error will be returned. +func (s *MemcachedJumpHashSelector) Each(f func(net.Addr) error) error { + return s.servers.Each(f) +} diff --git a/pkg/cacheutil/memcached_server_selector_test.go b/pkg/cacheutil/memcached_server_selector_test.go new file mode 100644 index 0000000000..a215604ece --- /dev/null +++ b/pkg/cacheutil/memcached_server_selector_test.go @@ -0,0 +1,184 @@ +package cacheutil + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/bradfitz/gomemcache/memcache" + "github.com/facette/natsort" + "github.com/fortytw2/leaktest" + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestNatSort(t *testing.T) { + + // Validate that the order of SRV records returned by a DNS + // lookup for a k8s StatefulSet are ordered as expected when + // a natsort is done. + input := []string{ + "memcached-10.memcached.thanos.svc.cluster.local.", + "memcached-1.memcached.thanos.svc.cluster.local.", + "memcached-6.memcached.thanos.svc.cluster.local.", + "memcached-3.memcached.thanos.svc.cluster.local.", + "memcached-25.memcached.thanos.svc.cluster.local.", + } + + expected := []string{ + "memcached-1.memcached.thanos.svc.cluster.local.", + "memcached-3.memcached.thanos.svc.cluster.local.", + "memcached-6.memcached.thanos.svc.cluster.local.", + "memcached-10.memcached.thanos.svc.cluster.local.", + "memcached-25.memcached.thanos.svc.cluster.local.", + } + + natsort.Sort(input) + testutil.Equals(t, expected, input) +} + +func TestMemcachedJumpHashSelector_Each_ShouldRespectServersOrdering(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + tests := []struct { + input []string + expected []string + }{ + { + input: []string{"127.0.0.1:11211", "127.0.0.2:11211", "127.0.0.3:11211"}, + expected: []string{"127.0.0.1:11211", "127.0.0.2:11211", "127.0.0.3:11211"}, + }, + { + input: []string{"127.0.0.2:11211", "127.0.0.3:11211", "127.0.0.1:11211"}, + expected: []string{"127.0.0.1:11211", "127.0.0.2:11211", "127.0.0.3:11211"}, + }, + } + + s := MemcachedJumpHashSelector{} + + for _, test := range tests { + testutil.Ok(t, s.SetServers(test.input...)) + + actual := make([]string, 0, 3) + err := s.Each(func(addr net.Addr) error { + actual = append(actual, addr.String()) + return nil + }) + + testutil.Ok(t, err) + testutil.Equals(t, test.expected, actual) + } +} + +func TestMemcachedJumpHashSelector_PickServer_ShouldEvenlyDistributeKeysToServers(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + servers := []string{"127.0.0.1:11211", "127.0.0.2:11211", "127.0.0.3:11211"} + selector := MemcachedJumpHashSelector{} + testutil.Ok(t, selector.SetServers(servers...)) + + // Calculate the distribution of keys. + distribution := make(map[string]int) + + for i := 0; i < 1000; i++ { + key := fmt.Sprintf("key-%d", i) + addr, err := selector.PickServer(key) + testutil.Ok(t, err) + distribution[addr.String()]++ + } + + // Expect each server got at least 25% of keys, where the perfect split would be 33.3% each. + minKeysPerServer := int(float64(len(servers)) * 0.25) + testutil.Equals(t, len(servers), len(distribution)) + + for addr, count := range distribution { + if count < minKeysPerServer { + testutil.Ok(t, fmt.Errorf("expected %s to have received at least %d keys instead it received %d", addr, minKeysPerServer, count)) + } + } +} + +func TestMemcachedJumpHashSelector_PickServer_ShouldUseConsistentHashing(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + servers := []string{ + "127.0.0.1:11211", + "127.0.0.2:11211", + "127.0.0.3:11211", + "127.0.0.4:11211", + "127.0.0.5:11211", + "127.0.0.6:11211", + "127.0.0.7:11211", + "127.0.0.8:11211", + "127.0.0.9:11211", + } + + selector := MemcachedJumpHashSelector{} + testutil.Ok(t, selector.SetServers(servers...)) + + // Pick a server for each key. + distribution := make(map[string]string) + numKeys := 1000 + + for i := 0; i < 1000; i++ { + key := fmt.Sprintf("key-%d", i) + addr, err := selector.PickServer(key) + testutil.Ok(t, err) + distribution[key] = addr.String() + } + + // Add 1 more server that - in a natural ordering - is added as last. + servers = append(servers, "127.0.0.10:11211") + testutil.Ok(t, selector.SetServers(servers...)) + + // Calculate the number of keys who has been moved due to the resharding. + moved := 0 + + for i := 0; i < 1000; i++ { + key := fmt.Sprintf("key-%d", i) + addr, err := selector.PickServer(key) + testutil.Ok(t, err) + + if distribution[key] != addr.String() { + moved++ + } + } + + // Expect we haven't moved more than (1/shards)% +2% tolerance. + maxExpectedMovedPerc := (1.0 / float64(len(servers))) + 0.02 + maxExpectedMoved := int(float64(numKeys) * maxExpectedMovedPerc) + if moved > maxExpectedMoved { + testutil.Ok(t, fmt.Errorf("expected resharding moved no more then %d keys while %d have been moved", maxExpectedMoved, moved)) + } +} + +func TestMemcachedJumpHashSelector_PickServer_ShouldReturnErrNoServersOnNoServers(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + s := MemcachedJumpHashSelector{} + _, err := s.PickServer("foo") + testutil.Equals(t, memcache.ErrNoServers, err) +} + +func BenchmarkMemcachedJumpHashSelector_PickServer(b *testing.B) { + // Create a pretty long list of servers. + servers := make([]string, 0) + for i := 1; i <= 60; i++ { + servers = append(servers, fmt.Sprintf("127.0.0.%d:11211", i)) + } + + selector := MemcachedJumpHashSelector{} + err := selector.SetServers(servers...) + if err != nil { + b.Error(err) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := selector.PickServer(string(i)) + if err != nil { + b.Error(err) + } + } +} diff --git a/pkg/store/gate.go b/pkg/gate/gate.go similarity index 99% rename from pkg/store/gate.go rename to pkg/gate/gate.go index 7662e5392f..78aeefe671 100644 --- a/pkg/store/gate.go +++ b/pkg/gate/gate.go @@ -1,4 +1,4 @@ -package store +package gate import ( "context" diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index d82690d7c3..ddb9edb114 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -32,6 +32,7 @@ import ( "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extprom" + "github.com/thanos-io/thanos/pkg/gate" "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/pool" @@ -215,7 +216,7 @@ type BucketStore struct { blockSyncConcurrency int // Query gate which limits the maximum amount of concurrent queries. - queryGate *Gate + queryGate *gate.Gate // samplesLimiter limits the number of samples per each Series() call. samplesLimiter *Limiter @@ -271,7 +272,7 @@ func NewBucketStore( blockSets: map[uint64]*bucketBlockSet{}, debugLogging: debugLogging, blockSyncConcurrency: blockSyncConcurrency, - queryGate: NewGate( + queryGate: gate.NewGate( maxConcurrent, extprom.WrapRegistererWithPrefix("thanos_bucket_store_series_", reg), ), @@ -1527,7 +1528,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { keys = append(keys, g.keys...) } - fromCache, _ := r.cache.FetchMultiPostings(r.block.meta.ULID, keys) + fromCache, _ := r.cache.FetchMultiPostings(r.ctx, r.block.meta.ULID, keys) // Iterate over all groups and fetch posting from cache. // If we have a miss, mark key to be fetched in `ptrs` slice. @@ -1606,7 +1607,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { // Return postings and fill LRU cache. groups[p.groupID].Fill(p.keyID, fetchedPostings) - r.cache.StorePostings(r.block.meta.ULID, groups[p.groupID].keys[p.keyID], c) + r.cache.StorePostings(r.ctx, r.block.meta.ULID, groups[p.groupID].keys[p.keyID], c) // If we just fetched it we still have to update the stats for touched postings. r.stats.postingsTouched++ @@ -1624,7 +1625,7 @@ func (r *bucketIndexReader) PreloadSeries(ids []uint64) error { // Load series from cache, overwriting the list of ids to preload // with the missing ones. - fromCache, ids := r.cache.FetchMultiSeries(r.block.meta.ULID, ids) + fromCache, ids := r.cache.FetchMultiSeries(r.ctx, r.block.meta.ULID, ids) for id, b := range fromCache { r.loadedSeries[id] = b } @@ -1672,7 +1673,7 @@ func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, start, } c = c[n : n+int(l)] r.loadedSeries[id] = c - r.cache.StoreSeries(r.block.meta.ULID, id, c) + r.cache.StoreSeries(r.ctx, r.block.meta.ULID, id, c) } return nil } diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index c30ec4e36d..d860887b3a 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -37,13 +37,13 @@ var ( type noopCache struct{} -func (noopCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte) {} -func (noopCache) FetchMultiPostings(blockID ulid.ULID, keys []labels.Label) (map[labels.Label][]byte, []labels.Label) { +func (noopCache) StorePostings(ctx context.Context, blockID ulid.ULID, l labels.Label, v []byte) {} +func (noopCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label) (map[labels.Label][]byte, []labels.Label) { return map[labels.Label][]byte{}, keys } -func (noopCache) StoreSeries(blockID ulid.ULID, id uint64, v []byte) {} -func (noopCache) FetchMultiSeries(blockID ulid.ULID, ids []uint64) (map[uint64][]byte, []uint64) { +func (noopCache) StoreSeries(ctx context.Context, blockID ulid.ULID, id uint64, v []byte) {} +func (noopCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []uint64) (map[uint64][]byte, []uint64) { return map[uint64][]byte{}, ids } @@ -55,20 +55,20 @@ func (c *swappableCache) SwapWith(ptr2 storecache.IndexCache) { c.ptr = ptr2 } -func (c *swappableCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte) { - c.ptr.StorePostings(blockID, l, v) +func (c *swappableCache) StorePostings(ctx context.Context, blockID ulid.ULID, l labels.Label, v []byte) { + c.ptr.StorePostings(ctx, blockID, l, v) } -func (c *swappableCache) FetchMultiPostings(blockID ulid.ULID, keys []labels.Label) (map[labels.Label][]byte, []labels.Label) { - return c.ptr.FetchMultiPostings(blockID, keys) +func (c *swappableCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label) (map[labels.Label][]byte, []labels.Label) { + return c.ptr.FetchMultiPostings(ctx, blockID, keys) } -func (c *swappableCache) StoreSeries(blockID ulid.ULID, id uint64, v []byte) { - c.ptr.StoreSeries(blockID, id, v) +func (c *swappableCache) StoreSeries(ctx context.Context, blockID ulid.ULID, id uint64, v []byte) { + c.ptr.StoreSeries(ctx, blockID, id, v) } -func (c *swappableCache) FetchMultiSeries(blockID ulid.ULID, ids []uint64) (map[uint64][]byte, []uint64) { - return c.ptr.FetchMultiSeries(blockID, ids) +func (c *swappableCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []uint64) (map[uint64][]byte, []uint64) { + return c.ptr.FetchMultiSeries(ctx, blockID, ids) } type storeSuite struct { @@ -378,18 +378,18 @@ func TestBucketStore_e2e(t *testing.T) { testBucketStore_e2e(t, ctx, s) t.Log("Test with large, sufficient index cache") - indexCache, err := storecache.NewInMemoryIndexCache(s.logger, nil, storecache.Opts{ - MaxItemSizeBytes: 1e5, - MaxSizeBytes: 2e5, + indexCache, err := storecache.NewInMemoryIndexCacheWithConfig(s.logger, nil, storecache.InMemoryIndexCacheConfig{ + MaxItemSize: 1e5, + MaxSize: 2e5, }) testutil.Ok(t, err) s.cache.SwapWith(indexCache) testBucketStore_e2e(t, ctx, s) t.Log("Test with small index cache") - indexCache2, err := storecache.NewInMemoryIndexCache(s.logger, nil, storecache.Opts{ - MaxItemSizeBytes: 50, - MaxSizeBytes: 100, + indexCache2, err := storecache.NewInMemoryIndexCacheWithConfig(s.logger, nil, storecache.InMemoryIndexCacheConfig{ + MaxItemSize: 50, + MaxSize: 100, }) testutil.Ok(t, err) s.cache.SwapWith(indexCache2) @@ -421,9 +421,9 @@ func TestBucketStore_ManyParts_e2e(t *testing.T) { s := prepareStoreWithTestBlocks(t, dir, bkt, true, 0, emptyRelabelConfig) - indexCache, err := storecache.NewInMemoryIndexCache(s.logger, nil, storecache.Opts{ - MaxItemSizeBytes: 1e5, - MaxSizeBytes: 2e5, + indexCache, err := storecache.NewInMemoryIndexCacheWithConfig(s.logger, nil, storecache.InMemoryIndexCacheConfig{ + MaxItemSize: 1e5, + MaxSize: 2e5, }) testutil.Ok(t, err) s.cache.SwapWith(indexCache) diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index d2c42da88b..d7e175da82 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -1,22 +1,83 @@ package storecache import ( + "context" + "encoding/base64" + "strconv" + "github.com/oklog/ulid" "github.com/prometheus/prometheus/pkg/labels" + "golang.org/x/crypto/blake2b" +) + +const ( + cacheTypePostings string = "Postings" + cacheTypeSeries string = "Series" + + sliceHeaderSize = 16 ) +var ( + ulidSize = uint64(len(ulid.ULID{})) +) + +// IndexCache is the interface exported by index cache backends. type IndexCache interface { // StorePostings stores postings for a single series. - StorePostings(blockID ulid.ULID, l labels.Label, v []byte) + StorePostings(ctx context.Context, blockID ulid.ULID, l labels.Label, v []byte) // FetchMultiPostings fetches multiple postings - each identified by a label - // and returns a map containing cache hits, along with a list of missing keys. - FetchMultiPostings(blockID ulid.ULID, keys []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) + FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) // StoreSeries stores a single series. - StoreSeries(blockID ulid.ULID, id uint64, v []byte) + StoreSeries(ctx context.Context, blockID ulid.ULID, id uint64, v []byte) // FetchMultiSeries fetches multiple series - each identified by ID - from the cache // and returns a map containing cache hits, along with a list of missing IDs. - FetchMultiSeries(blockID ulid.ULID, ids []uint64) (hits map[uint64][]byte, misses []uint64) + FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []uint64) (hits map[uint64][]byte, misses []uint64) +} + +type cacheKey struct { + block ulid.ULID + key interface{} } + +func (c cacheKey) keyType() string { + switch c.key.(type) { + case cacheKeyPostings: + return cacheTypePostings + case cacheKeySeries: + return cacheTypeSeries + } + return "" +} + +func (c cacheKey) size() uint64 { + switch k := c.key.(type) { + case cacheKeyPostings: + // ULID + 2 slice headers + number of chars in value and name. + return ulidSize + 2*sliceHeaderSize + uint64(len(k.Value)+len(k.Name)) + case cacheKeySeries: + return ulidSize + 8 // ULID + uint64. + } + return 0 +} + +func (c cacheKey) string() string { + switch c.key.(type) { + case cacheKeyPostings: + // Use cryptographically hash functions to avoid hash collisions + // which would end up in wrong query results. + lbl := c.key.(cacheKeyPostings) + lblHash := blake2b.Sum256([]byte(lbl.Name + ":" + lbl.Value)) + return "P:" + c.block.String() + ":" + base64.RawURLEncoding.EncodeToString(lblHash[0:]) + case cacheKeySeries: + return "S:" + c.block.String() + ":" + strconv.FormatUint(uint64(c.key.(cacheKeySeries)), 10) + default: + return "" + } +} + +type cacheKeyPostings labels.Label +type cacheKeySeries uint64 diff --git a/pkg/store/cache/cache_test.go b/pkg/store/cache/cache_test.go new file mode 100644 index 0000000000..60b0a3db7f --- /dev/null +++ b/pkg/store/cache/cache_test.go @@ -0,0 +1,99 @@ +package storecache + +import ( + "encoding/base64" + "fmt" + "math" + "strings" + "testing" + + "github.com/oklog/ulid" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/thanos-io/thanos/pkg/testutil" + "golang.org/x/crypto/blake2b" +) + +func TestCacheKey_string(t *testing.T) { + t.Parallel() + + uid := ulid.MustNew(1, nil) + + tests := map[string]struct { + key cacheKey + expected string + }{ + "should stringify postings cache key": { + key: cacheKey{uid, cacheKeyPostings(labels.Label{Name: "foo", Value: "bar"})}, + expected: func() string { + hash := blake2b.Sum256([]byte("foo:bar")) + encodedHash := base64.RawURLEncoding.EncodeToString(hash[0:]) + + return fmt.Sprintf("P:%s:%s", uid.String(), encodedHash) + }(), + }, + "should stringify series cache key": { + key: cacheKey{uid, cacheKeySeries(12345)}, + expected: fmt.Sprintf("S:%s:12345", uid.String()), + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + actual := testData.key.string() + testutil.Equals(t, testData.expected, actual) + }) + } +} + +func TestCacheKey_string_ShouldGuaranteeReasonablyShortKeyLength(t *testing.T) { + t.Parallel() + + uid := ulid.MustNew(1, nil) + + tests := map[string]struct { + keys []cacheKey + expectedLen int + }{ + "should guarantee reasonably short key length for postings": { + expectedLen: 72, + keys: []cacheKey{ + {uid, cacheKeyPostings(labels.Label{Name: "a", Value: "b"})}, + {uid, cacheKeyPostings(labels.Label{Name: strings.Repeat("a", 100), Value: strings.Repeat("a", 1000)})}, + }, + }, + "should guarantee reasonably short key length for series": { + expectedLen: 49, + keys: []cacheKey{ + {uid, cacheKeySeries(math.MaxUint64)}, + }, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + for _, key := range testData.keys { + testutil.Equals(t, testData.expectedLen, len(key.string())) + } + }) + } +} + +func BenchmarkCacheKey_string_Postings(b *testing.B) { + uid := ulid.MustNew(1, nil) + key := cacheKey{uid, cacheKeyPostings(labels.Label{Name: strings.Repeat("a", 100), Value: strings.Repeat("a", 1000)})} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + key.string() + } +} + +func BenchmarkCacheKey_string_Series(b *testing.B) { + uid := ulid.MustNew(1, nil) + key := cacheKey{uid, cacheKeySeries(math.MaxUint64)} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + key.string() + } +} diff --git a/pkg/store/cache/factory.go b/pkg/store/cache/factory.go new file mode 100644 index 0000000000..7fed27dac2 --- /dev/null +++ b/pkg/store/cache/factory.go @@ -0,0 +1,58 @@ +package storecache + +import ( + "fmt" + "strings" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/thanos-io/thanos/pkg/cacheutil" + "gopkg.in/yaml.v2" +) + +type IndexCacheProvider string + +const ( + INMEMORY IndexCacheProvider = "IN-MEMORY" + MEMCACHED IndexCacheProvider = "MEMCACHED" +) + +// IndexCacheConfig specifies the index cache config. +type IndexCacheConfig struct { + Type IndexCacheProvider `yaml:"type"` + Config interface{} `yaml:"config"` +} + +// NewIndexCache initializes and returns new index cache. +func NewIndexCache(logger log.Logger, confContentYaml []byte, reg prometheus.Registerer) (IndexCache, error) { + level.Info(logger).Log("msg", "loading index cache configuration") + cacheConfig := &IndexCacheConfig{} + if err := yaml.UnmarshalStrict(confContentYaml, cacheConfig); err != nil { + return nil, errors.Wrap(err, "parsing config YAML file") + } + + backendConfig, err := yaml.Marshal(cacheConfig.Config) + if err != nil { + return nil, errors.Wrap(err, "marshal content of cache backend configuration") + } + + var cache IndexCache + switch strings.ToUpper(string(cacheConfig.Type)) { + case string(INMEMORY): + cache, err = NewInMemoryIndexCache(logger, reg, backendConfig) + case string(MEMCACHED): + var memcached cacheutil.MemcachedClient + memcached, err = cacheutil.NewMemcachedClient(logger, "index-cache", backendConfig, reg) + if err == nil { + cache, err = NewMemcachedIndexCache(logger, memcached, reg) + } + default: + return nil, errors.Errorf("index cache with type %s is not supported", cacheConfig.Type) + } + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("create %s index cache", cacheConfig.Type)) + } + return cache, nil +} diff --git a/pkg/store/cache/inmemory.go b/pkg/store/cache/inmemory.go index 3ef7d0c7ce..3b8881eb65 100644 --- a/pkg/store/cache/inmemory.go +++ b/pkg/store/cache/inmemory.go @@ -1,6 +1,7 @@ package storecache import ( + "context" "math" "sync" @@ -11,43 +12,15 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" + "gopkg.in/yaml.v2" ) -const ( - cacheTypePostings string = "Postings" - cacheTypeSeries string = "Series" - - sliceHeaderSize = 16 -) - -type cacheKey struct { - block ulid.ULID - key interface{} -} - -func (c cacheKey) keyType() string { - switch c.key.(type) { - case cacheKeyPostings: - return cacheTypePostings - case cacheKeySeries: - return cacheTypeSeries +var ( + DefaultInMemoryIndexCacheConfig = InMemoryIndexCacheConfig{ + MaxSize: 250 * 1024 * 1024, + MaxItemSize: 125 * 1024 * 1024, } - return "" -} - -func (c cacheKey) size() uint64 { - switch k := c.key.(type) { - case cacheKeyPostings: - // ULID + 2 slice headers + number of chars in value and name. - return 16 + 2*sliceHeaderSize + uint64(len(k.Value)+len(k.Name)) - case cacheKeySeries: - return 16 + 8 // ULID + uint64. - } - return 0 -} - -type cacheKeyPostings labels.Label -type cacheKeySeries uint64 +) type InMemoryIndexCache struct { mtx sync.Mutex @@ -69,24 +42,46 @@ type InMemoryIndexCache struct { overflow *prometheus.CounterVec } -type Opts struct { - // MaxSizeBytes represents overall maximum number of bytes cache can contain. - MaxSizeBytes uint64 - // MaxItemSizeBytes represents maximum size of single item. - MaxItemSizeBytes uint64 +// InMemoryIndexCacheConfig holds the in-memory index cache config. +type InMemoryIndexCacheConfig struct { + // MaxSize represents overall maximum number of bytes cache can contain. + MaxSize Bytes `yaml:"max_size"` + // MaxItemSize represents maximum size of single item. + MaxItemSize Bytes `yaml:"max_item_size"` +} + +// parseInMemoryIndexCacheConfig unmarshals a buffer into a InMemoryIndexCacheConfig with default values. +func parseInMemoryIndexCacheConfig(conf []byte) (InMemoryIndexCacheConfig, error) { + config := DefaultInMemoryIndexCacheConfig + if err := yaml.Unmarshal(conf, &config); err != nil { + return InMemoryIndexCacheConfig{}, err + } + + return config, nil } // NewInMemoryIndexCache creates a new thread-safe LRU cache for index entries and ensures the total cache // size approximately does not exceed maxBytes. -func NewInMemoryIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*InMemoryIndexCache, error) { - if opts.MaxItemSizeBytes > opts.MaxSizeBytes { - return nil, errors.Errorf("max item size (%v) cannot be bigger than overall cache size (%v)", opts.MaxItemSizeBytes, opts.MaxSizeBytes) +func NewInMemoryIndexCache(logger log.Logger, reg prometheus.Registerer, conf []byte) (*InMemoryIndexCache, error) { + config, err := parseInMemoryIndexCacheConfig(conf) + if err != nil { + return nil, err + } + + return NewInMemoryIndexCacheWithConfig(logger, reg, config) +} + +// NewInMemoryIndexCacheWithConfig creates a new thread-safe LRU cache for index entries and ensures the total cache +// size approximately does not exceed maxBytes. +func NewInMemoryIndexCacheWithConfig(logger log.Logger, reg prometheus.Registerer, config InMemoryIndexCacheConfig) (*InMemoryIndexCache, error) { + if config.MaxItemSize > config.MaxSize { + return nil, errors.Errorf("max item size (%v) cannot be bigger than overall cache size (%v)", config.MaxItemSize, config.MaxSize) } c := &InMemoryIndexCache{ logger: logger, - maxSizeBytes: opts.MaxSizeBytes, - maxItemSizeBytes: opts.MaxItemSizeBytes, + maxSizeBytes: uint64(config.MaxSize), + maxItemSizeBytes: uint64(config.MaxItemSize), } c.evicted = prometheus.NewCounterVec(prometheus.CounterOpts{ @@ -170,7 +165,7 @@ func NewInMemoryIndexCache(logger log.Logger, reg prometheus.Registerer, opts Op c.lru = l level.Info(logger).Log( - "msg", "created index cache", + "msg", "created in-memory index cache", "maxItemSizeBytes", c.maxItemSizeBytes, "maxSizeBytes", c.maxSizeBytes, "maxItems", "math.MaxInt64", @@ -273,13 +268,13 @@ func (c *InMemoryIndexCache) reset() { // StorePostings sets the postings identified by the ulid and label to the value v, // if the postings already exists in the cache it is not mutated. -func (c *InMemoryIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte) { +func (c *InMemoryIndexCache) StorePostings(ctx context.Context, blockID ulid.ULID, l labels.Label, v []byte) { c.set(cacheTypePostings, cacheKey{blockID, cacheKeyPostings(l)}, v) } // FetchMultiPostings fetches multiple postings - each identified by a label - // and returns a map containing cache hits, along with a list of missing keys. -func (c *InMemoryIndexCache) FetchMultiPostings(blockID ulid.ULID, keys []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) { +func (c *InMemoryIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) { hits = map[labels.Label][]byte{} for _, key := range keys { @@ -296,13 +291,13 @@ func (c *InMemoryIndexCache) FetchMultiPostings(blockID ulid.ULID, keys []labels // StoreSeries sets the series identified by the ulid and id to the value v, // if the series already exists in the cache it is not mutated. -func (c *InMemoryIndexCache) StoreSeries(blockID ulid.ULID, id uint64, v []byte) { +func (c *InMemoryIndexCache) StoreSeries(ctx context.Context, blockID ulid.ULID, id uint64, v []byte) { c.set(cacheTypeSeries, cacheKey{blockID, cacheKeySeries(id)}, v) } // FetchMultiSeries fetches multiple series - each identified by ID - from the cache // and returns a map containing cache hits, along with a list of missing IDs. -func (c *InMemoryIndexCache) FetchMultiSeries(blockID ulid.ULID, ids []uint64) (hits map[uint64][]byte, misses []uint64) { +func (c *InMemoryIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []uint64) (hits map[uint64][]byte, misses []uint64) { hits = map[uint64][]byte{} for _, id := range ids { diff --git a/pkg/store/cache/inmemory_test.go b/pkg/store/cache/inmemory_test.go index f70c79bf50..e14d3794af 100644 --- a/pkg/store/cache/inmemory_test.go +++ b/pkg/store/cache/inmemory_test.go @@ -3,6 +3,7 @@ package storecache import ( "bytes" + "context" "fmt" "math" "testing" @@ -18,13 +19,41 @@ import ( "github.com/thanos-io/thanos/pkg/testutil" ) +func TestNewInMemoryIndexCache(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + // Should return error on invalid YAML config. + conf := []byte("invalid") + cache, err := NewInMemoryIndexCache(log.NewNopLogger(), nil, conf) + testutil.NotOk(t, err) + testutil.Equals(t, (*InMemoryIndexCache)(nil), cache) + + // Should instance an in-memory index cache with default config + // on empty YAML config. + conf = []byte{} + cache, err = NewInMemoryIndexCache(log.NewNopLogger(), nil, conf) + testutil.Ok(t, err) + testutil.Equals(t, uint64(DefaultInMemoryIndexCacheConfig.MaxSize), cache.maxSizeBytes) + testutil.Equals(t, uint64(DefaultInMemoryIndexCacheConfig.MaxItemSize), cache.maxItemSizeBytes) + + // Should instance an in-memory index cache with specified YAML config.s with units. + conf = []byte(` +max_size: 1MB +max_item_size: 2KB +`) + cache, err = NewInMemoryIndexCache(log.NewNopLogger(), nil, conf) + testutil.Ok(t, err) + testutil.Equals(t, uint64(1024*1024), cache.maxSizeBytes) + testutil.Equals(t, uint64(2*1024), cache.maxItemSizeBytes) +} + func TestInMemoryIndexCache_AvoidsDeadlock(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() metrics := prometheus.NewRegistry() - cache, err := NewInMemoryIndexCache(log.NewNopLogger(), metrics, Opts{ - MaxItemSizeBytes: sliceHeaderSize + 5, - MaxSizeBytes: sliceHeaderSize + 5, + cache, err := NewInMemoryIndexCacheWithConfig(log.NewNopLogger(), metrics, InMemoryIndexCacheConfig{ + MaxItemSize: sliceHeaderSize + 5, + MaxSize: sliceHeaderSize + 5, }) testutil.Ok(t, err) @@ -37,14 +66,15 @@ func TestInMemoryIndexCache_AvoidsDeadlock(t *testing.T) { testutil.Ok(t, err) cache.lru = l - cache.StorePostings(ulid.MustNew(0, nil), labels.Label{Name: "test2", Value: "1"}, []byte{42, 33, 14, 67, 11}) + ctx := context.Background() + cache.StorePostings(ctx, ulid.MustNew(0, nil), labels.Label{Name: "test2", Value: "1"}, []byte{42, 33, 14, 67, 11}) testutil.Equals(t, uint64(sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(cache.curSize), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) // This triggers deadlock logic. - cache.StorePostings(ulid.MustNew(0, nil), labels.Label{Name: "test1", Value: "1"}, []byte{42}) + cache.StorePostings(ctx, ulid.MustNew(0, nil), labels.Label{Name: "test1", Value: "1"}, []byte{42}) testutil.Equals(t, uint64(sliceHeaderSize+1), cache.curSize) testutil.Equals(t, float64(cache.curSize), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) @@ -74,14 +104,15 @@ func TestInMemoryIndexCache_UpdateItem(t *testing.T) { }) metrics := prometheus.NewRegistry() - cache, err := NewInMemoryIndexCache(log.NewSyncLogger(errorLogger), metrics, Opts{ - MaxItemSizeBytes: maxSize, - MaxSizeBytes: maxSize, + cache, err := NewInMemoryIndexCacheWithConfig(log.NewSyncLogger(errorLogger), metrics, InMemoryIndexCacheConfig{ + MaxItemSize: maxSize, + MaxSize: maxSize, }) testutil.Ok(t, err) uid := func(id uint64) ulid.ULID { return ulid.MustNew(id, nil) } lbl := labels.Label{Name: "foo", Value: "bar"} + ctx := context.Background() for _, tt := range []struct { typ string @@ -90,9 +121,9 @@ func TestInMemoryIndexCache_UpdateItem(t *testing.T) { }{ { typ: cacheTypePostings, - set: func(id uint64, b []byte) { cache.StorePostings(uid(id), lbl, b) }, + set: func(id uint64, b []byte) { cache.StorePostings(ctx, uid(id), lbl, b) }, get: func(id uint64) ([]byte, bool) { - hits, _ := cache.FetchMultiPostings(uid(id), []labels.Label{lbl}) + hits, _ := cache.FetchMultiPostings(ctx, uid(id), []labels.Label{lbl}) b, ok := hits[lbl] return b, ok @@ -100,9 +131,9 @@ func TestInMemoryIndexCache_UpdateItem(t *testing.T) { }, { typ: cacheTypeSeries, - set: func(id uint64, b []byte) { cache.StoreSeries(uid(id), id, b) }, + set: func(id uint64, b []byte) { cache.StoreSeries(ctx, uid(id), id, b) }, get: func(id uint64) ([]byte, bool) { - hits, _ := cache.FetchMultiSeries(uid(id), []uint64{id}) + hits, _ := cache.FetchMultiSeries(ctx, uid(id), []uint64{id}) b, ok := hits[id] return b, ok @@ -159,9 +190,9 @@ func TestInMemoryIndexCache_MaxNumberOfItemsHit(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() metrics := prometheus.NewRegistry() - cache, err := NewInMemoryIndexCache(log.NewNopLogger(), metrics, Opts{ - MaxItemSizeBytes: 2*sliceHeaderSize + 10, - MaxSizeBytes: 2*sliceHeaderSize + 10, + cache, err := NewInMemoryIndexCacheWithConfig(log.NewNopLogger(), metrics, InMemoryIndexCacheConfig{ + MaxItemSize: 2*sliceHeaderSize + 10, + MaxSize: 2*sliceHeaderSize + 10, }) testutil.Ok(t, err) @@ -170,10 +201,11 @@ func TestInMemoryIndexCache_MaxNumberOfItemsHit(t *testing.T) { cache.lru = l id := ulid.MustNew(0, nil) + ctx := context.Background() - cache.StorePostings(id, labels.Label{Name: "test", Value: "123"}, []byte{42, 33}) - cache.StorePostings(id, labels.Label{Name: "test", Value: "124"}, []byte{42, 33}) - cache.StorePostings(id, labels.Label{Name: "test", Value: "125"}, []byte{42, 33}) + cache.StorePostings(ctx, id, labels.Label{Name: "test", Value: "123"}, []byte{42, 33}) + cache.StorePostings(ctx, id, labels.Label{Name: "test", Value: "124"}, []byte{42, 33}) + cache.StorePostings(ctx, id, labels.Label{Name: "test", Value: "125"}, []byte{42, 33}) testutil.Equals(t, uint64(2*sliceHeaderSize+4), cache.curSize) testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) @@ -192,25 +224,26 @@ func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() metrics := prometheus.NewRegistry() - cache, err := NewInMemoryIndexCache(log.NewNopLogger(), metrics, Opts{ - MaxItemSizeBytes: 2*sliceHeaderSize + 5, - MaxSizeBytes: 2*sliceHeaderSize + 5, + cache, err := NewInMemoryIndexCacheWithConfig(log.NewNopLogger(), metrics, InMemoryIndexCacheConfig{ + MaxItemSize: 2*sliceHeaderSize + 5, + MaxSize: 2*sliceHeaderSize + 5, }) testutil.Ok(t, err) id := ulid.MustNew(0, nil) lbls := labels.Label{Name: "test", Value: "123"} + ctx := context.Background() emptyPostingsHits := map[labels.Label][]byte{} emptyPostingsMisses := []labels.Label(nil) emptySeriesHits := map[uint64][]byte{} emptySeriesMisses := []uint64(nil) - pHits, pMisses := cache.FetchMultiPostings(id, []labels.Label{lbls}) + pHits, pMisses := cache.FetchMultiPostings(ctx, id, []labels.Label{lbls}) testutil.Equals(t, emptyPostingsHits, pHits, "no such key") testutil.Equals(t, []labels.Label{lbls}, pMisses) // Add sliceHeaderSize + 2 bytes. - cache.StorePostings(id, lbls, []byte{42, 33}) + cache.StorePostings(ctx, id, lbls, []byte{42, 33}) testutil.Equals(t, uint64(sliceHeaderSize+2), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(sliceHeaderSize+2), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) @@ -223,20 +256,20 @@ func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls}) + pHits, pMisses = cache.FetchMultiPostings(ctx, id, []labels.Label{lbls}) testutil.Equals(t, map[labels.Label][]byte{lbls: []byte{42, 33}}, pHits, "key exists") testutil.Equals(t, emptyPostingsMisses, pMisses) - pHits, pMisses = cache.FetchMultiPostings(ulid.MustNew(1, nil), []labels.Label{lbls}) + pHits, pMisses = cache.FetchMultiPostings(ctx, ulid.MustNew(1, nil), []labels.Label{lbls}) testutil.Equals(t, emptyPostingsHits, pHits, "no such key") testutil.Equals(t, []labels.Label{lbls}, pMisses) - pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{{Name: "test", Value: "124"}}) + pHits, pMisses = cache.FetchMultiPostings(ctx, id, []labels.Label{{Name: "test", Value: "124"}}) testutil.Equals(t, emptyPostingsHits, pHits, "no such key") testutil.Equals(t, []labels.Label{{Name: "test", Value: "124"}}, pMisses) // Add sliceHeaderSize + 3 more bytes. - cache.StoreSeries(id, 1234, []byte{222, 223, 224}) + cache.StoreSeries(ctx, id, 1234, []byte{222, 223, 224}) testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(sliceHeaderSize+2), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) @@ -249,7 +282,7 @@ func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - sHits, sMisses := cache.FetchMultiSeries(id, []uint64{1234}) + sHits, sMisses := cache.FetchMultiSeries(ctx, id, []uint64{1234}) testutil.Equals(t, map[uint64][]byte{1234: []byte{222, 223, 224}}, sHits, "key exists") testutil.Equals(t, emptySeriesMisses, sMisses) @@ -260,7 +293,7 @@ func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { for i := 0; i < sliceHeaderSize; i++ { v = append(v, 3) } - cache.StorePostings(id, lbls2, v) + cache.StorePostings(ctx, id, lbls2, v) testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) @@ -275,20 +308,20 @@ func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) // Eviction. // Evicted. - pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls}) + pHits, pMisses = cache.FetchMultiPostings(ctx, id, []labels.Label{lbls}) testutil.Equals(t, emptyPostingsHits, pHits, "no such key") testutil.Equals(t, []labels.Label{lbls}, pMisses) - sHits, sMisses = cache.FetchMultiSeries(id, []uint64{1234}) + sHits, sMisses = cache.FetchMultiSeries(ctx, id, []uint64{1234}) testutil.Equals(t, emptySeriesHits, sHits, "no such key") testutil.Equals(t, []uint64{1234}, sMisses) - pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls2}) + pHits, pMisses = cache.FetchMultiPostings(ctx, id, []labels.Label{lbls2}) testutil.Equals(t, map[labels.Label][]byte{lbls2: v}, pHits) testutil.Equals(t, emptyPostingsMisses, pMisses) // Add same item again. - cache.StorePostings(id, lbls2, v) + cache.StorePostings(ctx, id, lbls2, v) testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) @@ -302,12 +335,12 @@ func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls2}) + pHits, pMisses = cache.FetchMultiPostings(ctx, id, []labels.Label{lbls2}) testutil.Equals(t, map[labels.Label][]byte{lbls2: v}, pHits) testutil.Equals(t, emptyPostingsMisses, pMisses) // Add too big item. - cache.StorePostings(id, labels.Label{Name: "test", Value: "toobig"}, append(v, 5)) + cache.StorePostings(ctx, id, labels.Label{Name: "test", Value: "toobig"}, append(v, 5)) testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(2*sliceHeaderSize+5), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) @@ -340,7 +373,7 @@ func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { lbls3 := labels.Label{Name: "test", Value: "124"} - cache.StorePostings(id, lbls3, []byte{}) + cache.StorePostings(ctx, id, lbls3, []byte{}) testutil.Equals(t, uint64(sliceHeaderSize), cache.curSize) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) @@ -354,13 +387,13 @@ func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(2), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls3}) + pHits, pMisses = cache.FetchMultiPostings(ctx, id, []labels.Label{lbls3}) testutil.Equals(t, map[labels.Label][]byte{lbls3: []byte{}}, pHits, "key exists") testutil.Equals(t, emptyPostingsMisses, pMisses) // nil works and still allocates empty slice. lbls4 := labels.Label{Name: "test", Value: "125"} - cache.StorePostings(id, lbls4, []byte(nil)) + cache.StorePostings(ctx, id, lbls4, []byte(nil)) testutil.Equals(t, 2*uint64(sliceHeaderSize), cache.curSize) testutil.Equals(t, float64(2), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) @@ -374,7 +407,7 @@ func TestInMemoryIndexCache_Eviction_WithMetrics(t *testing.T) { testutil.Equals(t, float64(2), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) - pHits, pMisses = cache.FetchMultiPostings(id, []labels.Label{lbls4}) + pHits, pMisses = cache.FetchMultiPostings(ctx, id, []labels.Label{lbls4}) testutil.Equals(t, map[labels.Label][]byte{lbls4: []byte{}}, pHits, "key exists") testutil.Equals(t, emptyPostingsMisses, pMisses) diff --git a/pkg/store/cache/memcached.go b/pkg/store/cache/memcached.go new file mode 100644 index 0000000000..51c028dd54 --- /dev/null +++ b/pkg/store/cache/memcached.go @@ -0,0 +1,177 @@ +package storecache + +import ( + "context" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/oklog/ulid" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/thanos-io/thanos/pkg/cacheutil" +) + +const ( + memcachedDefaultTTL = 24 * time.Hour +) + +// MemcachedIndexCache is a memcached-based index cache. +type MemcachedIndexCache struct { + logger log.Logger + memcached cacheutil.MemcachedClient + + // Metrics. + requests *prometheus.CounterVec + hits *prometheus.CounterVec +} + +// NewMemcachedIndexCache makes a new MemcachedIndexCache. +func NewMemcachedIndexCache(logger log.Logger, memcached cacheutil.MemcachedClient, reg prometheus.Registerer) (*MemcachedIndexCache, error) { + c := &MemcachedIndexCache{ + logger: logger, + memcached: memcached, + } + + c.requests = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_requests_total", + Help: "Total number of items requests to the cache.", + }, []string{"item_type"}) + c.requests.WithLabelValues(cacheTypePostings) + c.requests.WithLabelValues(cacheTypeSeries) + + c.hits = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_hits_total", + Help: "Total number of items requests to the cache that were a hit.", + }, []string{"item_type"}) + c.hits.WithLabelValues(cacheTypePostings) + c.hits.WithLabelValues(cacheTypeSeries) + + if reg != nil { + reg.MustRegister(c.requests, c.hits) + } + + level.Info(logger).Log("msg", "created memcached index cache") + + return c, nil +} + +// StorePostings sets the postings identified by the ulid and label to the value v. +// The function enqueues the request and returns immediately: the entry will be +// asynchronously stored in the cache. +func (c *MemcachedIndexCache) StorePostings(ctx context.Context, blockID ulid.ULID, l labels.Label, v []byte) { + key := cacheKey{blockID, cacheKeyPostings(l)}.string() + + if err := c.memcached.SetAsync(ctx, key, v, memcachedDefaultTTL); err != nil { + level.Error(c.logger).Log("msg", "failed to cache postings in memcached", "err", err) + } +} + +// FetchMultiPostings fetches multiple postings - each identified by a label - +// and returns a map containing cache hits, along with a list of missing keys. +// In case of error, it logs and return an empty cache hits map. +func (c *MemcachedIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, lbls []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) { + // Build the cache keys, while keeping a map between input label and the cache key + // so that we can easily reverse it back after the GetMulti(). + keys := make([]string, 0, len(lbls)) + keysMapping := map[labels.Label]string{} + + for _, lbl := range lbls { + key := cacheKey{blockID, cacheKeyPostings(lbl)}.string() + + keys = append(keys, key) + keysMapping[lbl] = key + } + + // Fetch the keys from memcached in a single request. + c.requests.WithLabelValues(cacheTypePostings).Add(float64(len(keys))) + results := c.memcached.GetMulti(ctx, keys) + if len(results) == 0 { + return nil, lbls + } + + // Construct the resulting hits map and list of missing keys. We iterate on the input + // list of labels to be able to easily create the list of ones in a single iteration. + hits = map[labels.Label][]byte{} + + for _, lbl := range lbls { + key, ok := keysMapping[lbl] + if !ok { + level.Error(c.logger).Log("msg", "keys mapping inconsistency found in memcached index cache client", "type", "postings", "label", lbl.Name+":"+lbl.Value) + continue + } + + // Check if the key has been found in memcached. If not, we add it to the list + // of missing keys. + value, ok := results[key] + if !ok { + misses = append(misses, lbl) + continue + } + + hits[lbl] = value + } + + c.hits.WithLabelValues(cacheTypePostings).Add(float64(len(hits))) + return hits, misses +} + +// StoreSeries sets the series identified by the ulid and id to the value v. +// The function enqueues the request and returns immediately: the entry will be +// asynchronously stored in the cache. +func (c *MemcachedIndexCache) StoreSeries(ctx context.Context, blockID ulid.ULID, id uint64, v []byte) { + key := cacheKey{blockID, cacheKeySeries(id)}.string() + + if err := c.memcached.SetAsync(ctx, key, v, memcachedDefaultTTL); err != nil { + level.Error(c.logger).Log("msg", "failed to cache series in memcached", "err", err) + } +} + +// FetchMultiSeries fetches multiple series - each identified by ID - from the cache +// and returns a map containing cache hits, along with a list of missing IDs. +// In case of error, it logs and return an empty cache hits map. +func (c *MemcachedIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []uint64) (hits map[uint64][]byte, misses []uint64) { + // Build the cache keys, while keeping a map between input id and the cache key + // so that we can easily reverse it back after the GetMulti(). + keys := make([]string, 0, len(ids)) + keysMapping := map[uint64]string{} + + for _, id := range ids { + key := cacheKey{blockID, cacheKeySeries(id)}.string() + + keys = append(keys, key) + keysMapping[id] = key + } + + // Fetch the keys from memcached in a single request. + c.requests.WithLabelValues(cacheTypeSeries).Add(float64(len(ids))) + results := c.memcached.GetMulti(ctx, keys) + if len(results) == 0 { + return nil, ids + } + + // Construct the resulting hits map and list of missing keys. We iterate on the input + // list of ids to be able to easily create the list of ones in a single iteration. + hits = map[uint64][]byte{} + + for _, id := range ids { + key, ok := keysMapping[id] + if !ok { + level.Error(c.logger).Log("msg", "keys mapping inconsistency found in memcached index cache client", "type", "series", "id", id) + continue + } + + // Check if the key has been found in memcached. If not, we add it to the list + // of missing keys. + value, ok := results[key] + if !ok { + misses = append(misses, id) + continue + } + + hits[id] = value + } + + c.hits.WithLabelValues(cacheTypeSeries).Add(float64(len(hits))) + return hits, misses +} diff --git a/pkg/store/cache/memcached_test.go b/pkg/store/cache/memcached_test.go new file mode 100644 index 0000000000..83d858ff30 --- /dev/null +++ b/pkg/store/cache/memcached_test.go @@ -0,0 +1,247 @@ +package storecache + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/fortytw2/leaktest" + "github.com/go-kit/kit/log" + "github.com/oklog/ulid" + prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestMemcachedIndexCache_FetchMultiPostings(t *testing.T) { + t.Parallel() + defer leaktest.CheckTimeout(t, 10*time.Second)() + + // Init some data to conveniently define test cases later one. + block1 := ulid.MustNew(1, nil) + block2 := ulid.MustNew(2, nil) + label1 := labels.Label{Name: "instance", Value: "a"} + label2 := labels.Label{Name: "instance", Value: "b"} + value1 := []byte{1} + value2 := []byte{2} + value3 := []byte{3} + + tests := map[string]struct { + setup []mockedPostings + mockedErr error + fetchBlockID ulid.ULID + fetchLabels []labels.Label + expectedHits map[labels.Label][]byte + expectedMisses []labels.Label + }{ + "should return no hits on empty cache": { + setup: []mockedPostings{}, + fetchBlockID: block1, + fetchLabels: []labels.Label{label1, label2}, + expectedHits: nil, + expectedMisses: []labels.Label{label1, label2}, + }, + "should return no misses on 100% hit ratio": { + setup: []mockedPostings{ + {block: block1, label: label1, value: value1}, + {block: block1, label: label2, value: value2}, + {block: block2, label: label1, value: value3}, + }, + fetchBlockID: block1, + fetchLabels: []labels.Label{label1, label2}, + expectedHits: map[labels.Label][]byte{ + label1: value1, + label2: value2, + }, + expectedMisses: nil, + }, + "should return hits and misses on partial hits": { + setup: []mockedPostings{ + {block: block1, label: label1, value: value1}, + {block: block2, label: label1, value: value3}, + }, + fetchBlockID: block1, + fetchLabels: []labels.Label{label1, label2}, + expectedHits: map[labels.Label][]byte{label1: value1}, + expectedMisses: []labels.Label{label2}, + }, + "should return no hits on memcached error": { + setup: []mockedPostings{ + {block: block1, label: label1, value: value1}, + {block: block1, label: label2, value: value2}, + {block: block2, label: label1, value: value3}, + }, + mockedErr: errors.New("mocked error"), + fetchBlockID: block1, + fetchLabels: []labels.Label{label1, label2}, + expectedHits: nil, + expectedMisses: []labels.Label{label1, label2}, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + memcached := newMockedMemcachedClient(testData.mockedErr) + c, err := NewMemcachedIndexCache(log.NewNopLogger(), memcached, nil) + testutil.Ok(t, err) + + // Store the postings expected before running the test. + ctx := context.Background() + for _, p := range testData.setup { + c.StorePostings(ctx, p.block, p.label, p.value) + } + + // Fetch postings from cached and assert on it. + hits, misses := c.FetchMultiPostings(ctx, testData.fetchBlockID, testData.fetchLabels) + testutil.Equals(t, testData.expectedHits, hits) + testutil.Equals(t, testData.expectedMisses, misses) + + // Assert on metrics. + testutil.Equals(t, float64(len(testData.fetchLabels)), prom_testutil.ToFloat64(c.requests.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(len(testData.expectedHits)), prom_testutil.ToFloat64(c.hits.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, 0.0, prom_testutil.ToFloat64(c.requests.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, 0.0, prom_testutil.ToFloat64(c.hits.WithLabelValues(cacheTypeSeries))) + }) + } +} + +func TestMemcachedIndexCache_FetchMultiSeries(t *testing.T) { + t.Parallel() + defer leaktest.CheckTimeout(t, 10*time.Second)() + + // Init some data to conveniently define test cases later one. + block1 := ulid.MustNew(1, nil) + block2 := ulid.MustNew(2, nil) + value1 := []byte{1} + value2 := []byte{2} + value3 := []byte{3} + + tests := map[string]struct { + setup []mockedSeries + mockedErr error + fetchBlockID ulid.ULID + fetchIds []uint64 + expectedHits map[uint64][]byte + expectedMisses []uint64 + }{ + "should return no hits on empty cache": { + setup: []mockedSeries{}, + fetchBlockID: block1, + fetchIds: []uint64{1, 2}, + expectedHits: nil, + expectedMisses: []uint64{1, 2}, + }, + "should return no misses on 100% hit ratio": { + setup: []mockedSeries{ + {block: block1, id: 1, value: value1}, + {block: block1, id: 2, value: value2}, + {block: block2, id: 1, value: value3}, + }, + fetchBlockID: block1, + fetchIds: []uint64{1, 2}, + expectedHits: map[uint64][]byte{ + 1: value1, + 2: value2, + }, + expectedMisses: nil, + }, + "should return hits and misses on partial hits": { + setup: []mockedSeries{ + {block: block1, id: 1, value: value1}, + {block: block2, id: 1, value: value3}, + }, + fetchBlockID: block1, + fetchIds: []uint64{1, 2}, + expectedHits: map[uint64][]byte{1: value1}, + expectedMisses: []uint64{2}, + }, + "should return no hits on memcached error": { + setup: []mockedSeries{ + {block: block1, id: 1, value: value1}, + {block: block1, id: 2, value: value2}, + {block: block2, id: 1, value: value3}, + }, + mockedErr: errors.New("mocked error"), + fetchBlockID: block1, + fetchIds: []uint64{1, 2}, + expectedHits: nil, + expectedMisses: []uint64{1, 2}, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + memcached := newMockedMemcachedClient(testData.mockedErr) + c, err := NewMemcachedIndexCache(log.NewNopLogger(), memcached, nil) + testutil.Ok(t, err) + + // Store the series expected before running the test. + ctx := context.Background() + for _, p := range testData.setup { + c.StoreSeries(ctx, p.block, p.id, p.value) + } + + // Fetch series from cached and assert on it. + hits, misses := c.FetchMultiSeries(ctx, testData.fetchBlockID, testData.fetchIds) + testutil.Equals(t, testData.expectedHits, hits) + testutil.Equals(t, testData.expectedMisses, misses) + + // Assert on metrics. + testutil.Equals(t, float64(len(testData.fetchIds)), prom_testutil.ToFloat64(c.requests.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(len(testData.expectedHits)), prom_testutil.ToFloat64(c.hits.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, 0.0, prom_testutil.ToFloat64(c.requests.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, 0.0, prom_testutil.ToFloat64(c.hits.WithLabelValues(cacheTypePostings))) + }) + } +} + +type mockedPostings struct { + block ulid.ULID + label labels.Label + value []byte +} + +type mockedSeries struct { + block ulid.ULID + id uint64 + value []byte +} + +type mockedMemcachedClient struct { + cache map[string][]byte + mockedGetMultiErr error +} + +func newMockedMemcachedClient(mockedGetMultiErr error) *mockedMemcachedClient { + return &mockedMemcachedClient{ + cache: map[string][]byte{}, + mockedGetMultiErr: mockedGetMultiErr, + } +} + +func (c *mockedMemcachedClient) GetMulti(ctx context.Context, keys []string) map[string][]byte { + if c.mockedGetMultiErr != nil { + return nil + } + + hits := map[string][]byte{} + + for _, key := range keys { + if value, ok := c.cache[key]; ok { + hits[key] = value + } + } + + return hits +} + +func (c *mockedMemcachedClient) SetAsync(ctx context.Context, key string, value []byte, ttl time.Duration) error { + c.cache[key] = value + + return nil +} + +func (c *mockedMemcachedClient) Stop() { + // Nothing to do. +} diff --git a/pkg/store/cache/units.go b/pkg/store/cache/units.go new file mode 100644 index 0000000000..d9b6dce367 --- /dev/null +++ b/pkg/store/cache/units.go @@ -0,0 +1,29 @@ +package storecache + +import ( + "github.com/alecthomas/units" +) + +// Bytes is a data type which supports yaml serialization/deserialization +// with units. +type Bytes uint64 + +func (b *Bytes) UnmarshalYAML(unmarshal func(interface{}) error) error { + var value string + err := unmarshal(&value) + if err != nil { + return err + } + + bytes, err := units.ParseBase2Bytes(value) + if err != nil { + return err + } + + *b = Bytes(bytes) + return nil +} + +func (b *Bytes) MarshalYAML() (interface{}, error) { + return units.Base2Bytes(*b).String(), nil +} diff --git a/pkg/store/cache/units_test.go b/pkg/store/cache/units_test.go new file mode 100644 index 0000000000..95438eaa50 --- /dev/null +++ b/pkg/store/cache/units_test.go @@ -0,0 +1,21 @@ +package storecache + +import ( + "testing" + + "github.com/thanos-io/thanos/pkg/testutil" + "gopkg.in/yaml.v2" +) + +func TestBytes_Marshalling(t *testing.T) { + value := Bytes(1048576) + + encoded, err := yaml.Marshal(&value) + testutil.Ok(t, err) + testutil.Equals(t, "1MiB\n", string(encoded)) + + var decoded Bytes + err = yaml.Unmarshal(encoded, &decoded) + testutil.Equals(t, value, decoded) + testutil.Ok(t, err) +} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 2d5cec20e1..5f33f2bdba 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -13,6 +13,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/alert" + "github.com/thanos-io/thanos/pkg/cacheutil" "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/cos" @@ -21,6 +22,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" + storecache "github.com/thanos-io/thanos/pkg/store/cache" trclient "github.com/thanos-io/thanos/pkg/tracing/client" "github.com/thanos-io/thanos/pkg/tracing/elasticapm" "github.com/thanos-io/thanos/pkg/tracing/jaeger" @@ -46,6 +48,10 @@ var ( trclient.ELASTIC_APM: elasticapm.Config{}, trclient.LIGHTSTEP: lightstep.Config{}, } + indexCacheConfigs = map[storecache.IndexCacheProvider]interface{}{ + storecache.INMEMORY: storecache.InMemoryIndexCacheConfig{}, + storecache.MEMCACHED: cacheutil.MemcachedClientConfig{}, + } ) func main() { @@ -60,14 +66,21 @@ func main() { } for typ, config := range bucketConfigs { - if err := generate(client.BucketConfig{Type: typ, Config: config}, "bucket_"+strings.ToLower(string(typ)), *outputDir); err != nil { + if err := generate(client.BucketConfig{Type: typ, Config: config}, generateName("bucket_", string(typ)), *outputDir); err != nil { level.Error(logger).Log("msg", "failed to generate", "type", typ, "err", err) os.Exit(1) } } for typ, config := range tracingConfigs { - if err := generate(trclient.TracingConfig{Type: typ, Config: config}, "tracing_"+strings.ToLower(string(typ)), *outputDir); err != nil { + if err := generate(trclient.TracingConfig{Type: typ, Config: config}, generateName("tracing_", string(typ)), *outputDir); err != nil { + level.Error(logger).Log("msg", "failed to generate", "type", typ, "err", err) + os.Exit(1) + } + } + + for typ, config := range indexCacheConfigs { + if err := generate(storecache.IndexCacheConfig{Type: typ, Config: config}, generateName("index_cache_", string(typ)), *outputDir); err != nil { level.Error(logger).Log("msg", "failed to generate", "type", typ, "err", err) os.Exit(1) } @@ -96,6 +109,10 @@ func generate(obj interface{}, typ string, outputDir string) error { return ioutil.WriteFile(filepath.Join(outputDir, fmt.Sprintf("config_%s.txt", typ)), out, os.ModePerm) } +func generateName(prefix, typ string) string { + return prefix + strings.ReplaceAll(strings.ToLower(string(typ)), "-", "_") +} + func checkForOmitEmptyTagOption(obj interface{}) error { return checkForOmitEmptyTagOptionRec(reflect.ValueOf(obj)) } From d9764fb6d6b6a42db4794711852d7e919fc70415 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Fri, 3 Jan 2020 15:03:40 -0500 Subject: [PATCH 138/257] Added no-chunk option in series API (#1904) Signed-off-by: yeya24 add e2e tests Signed-off-by: yeya24 add tests for all stores Signed-off-by: yeya24 add changelog Signed-off-by: yeya24 --- .dep-finished | 0 CHANGELOG.md | 1 + pkg/query/api/v1.go | 10 +- pkg/query/querier.go | 12 +- pkg/query/querier_test.go | 6 +- pkg/store/bucket.go | 13 +- pkg/store/bucket_e2e_test.go | 34 ++++- pkg/store/matchers.go | 18 +++ pkg/store/matchers_test.go | 86 +++++++++++ pkg/store/prometheus.go | 109 +++++++++++++- pkg/store/prometheus_test.go | 100 ++++++++++++ pkg/store/proxy.go | 1 + pkg/store/proxy_test.go | 26 ++++ pkg/store/storepb/rpc.pb.go | 276 ++++++++++++++-------------------- pkg/store/storepb/rpc.proto | 3 + pkg/store/storepb/types.pb.go | 60 +++----- pkg/store/tsdb.go | 29 ++-- pkg/store/tsdb_test.go | 125 +++++++++++++++ 18 files changed, 664 insertions(+), 245 deletions(-) delete mode 100644 .dep-finished create mode 100644 pkg/store/matchers_test.go diff --git a/.dep-finished b/.dep-finished deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/CHANGELOG.md b/CHANGELOG.md index cab5a36852..e2a7706202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add TLS and authentication support for Alertmanager with the `--alertmanagers.config` and `--alertmanagers.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. - [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add a new `--alertmanagers.sd-dns-interval` CLI option to specify the interval between DNS resolutions of Alertmanager hosts. - [#1881](https://github.com/thanos-io/thanos/pull/1881) Store Gateway: memcached support for index cache. See [documentation](docs/components/store.md/#index-cache) for further information. +- [#1904](https://github.com/thanos-io/thanos/pull/1904) Add a skip-chunks option in Store Series API to improve the response time of `/api/v1/series` endpoint. ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 diff --git a/pkg/query/api/v1.go b/pkg/query/api/v1.go index 9dd9a9ec83..17ebc07321 100644 --- a/pkg/query/api/v1.go +++ b/pkg/query/api/v1.go @@ -293,7 +293,7 @@ func (api *API) query(r *http.Request) (interface{}, []error, *ApiError) { span, ctx := tracing.StartSpan(ctx, "promql_instant_query") defer span.Finish() - qry, err := api.queryEngine.NewInstantQuery(api.queryableCreate(enableDedup, replicaLabels, maxSourceResolution, enablePartialResponse), r.FormValue("query"), ts) + qry, err := api.queryEngine.NewInstantQuery(api.queryableCreate(enableDedup, replicaLabels, maxSourceResolution, enablePartialResponse, false), r.FormValue("query"), ts) if err != nil { return nil, nil, &ApiError{errorBadData, err} } @@ -386,7 +386,7 @@ func (api *API) queryRange(r *http.Request) (interface{}, []error, *ApiError) { defer span.Finish() qry, err := api.queryEngine.NewRangeQuery( - api.queryableCreate(enableDedup, replicaLabels, maxSourceResolution, enablePartialResponse), + api.queryableCreate(enableDedup, replicaLabels, maxSourceResolution, enablePartialResponse, false), r.FormValue("query"), start, end, @@ -426,7 +426,7 @@ func (api *API) labelValues(r *http.Request) (interface{}, []error, *ApiError) { return nil, nil, apiErr } - q, err := api.queryableCreate(true, nil, 0, enablePartialResponse).Querier(ctx, math.MinInt64, math.MaxInt64) + q, err := api.queryableCreate(true, nil, 0, enablePartialResponse, false).Querier(ctx, math.MinInt64, math.MaxInt64) if err != nil { return nil, nil, &ApiError{errorExec, err} } @@ -503,7 +503,7 @@ func (api *API) series(r *http.Request) (interface{}, []error, *ApiError) { } // TODO(bwplotka): Support downsampling? - q, err := api.queryableCreate(enableDedup, replicaLabels, 0, enablePartialResponse). + q, err := api.queryableCreate(enableDedup, replicaLabels, 0, enablePartialResponse, true). Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end)) if err != nil { return nil, nil, &ApiError{errorExec, err} @@ -607,7 +607,7 @@ func (api *API) labelNames(r *http.Request) (interface{}, []error, *ApiError) { return nil, nil, apiErr } - q, err := api.queryableCreate(true, nil, 0, enablePartialResponse).Querier(ctx, math.MinInt64, math.MaxInt64) + q, err := api.queryableCreate(true, nil, 0, enablePartialResponse, false).Querier(ctx, math.MinInt64, math.MaxInt64) if err != nil { return nil, nil, &ApiError{errorExec, err} } diff --git a/pkg/query/querier.go b/pkg/query/querier.go index 4263d81a36..a8294b00fa 100644 --- a/pkg/query/querier.go +++ b/pkg/query/querier.go @@ -19,11 +19,11 @@ import ( // replicaLabels at query time. // maxResolutionMillis controls downsampling resolution that is allowed (specified in milliseconds). // partialResponse controls `partialResponseDisabled` option of StoreAPI and partial response behaviour of proxy. -type QueryableCreator func(deduplicate bool, replicaLabels []string, maxResolutionMillis int64, partialResponse bool) storage.Queryable +type QueryableCreator func(deduplicate bool, replicaLabels []string, maxResolutionMillis int64, partialResponse, skipChunks bool) storage.Queryable // NewQueryableCreator creates QueryableCreator. func NewQueryableCreator(logger log.Logger, proxy storepb.StoreServer) QueryableCreator { - return func(deduplicate bool, replicaLabels []string, maxResolutionMillis int64, partialResponse bool) storage.Queryable { + return func(deduplicate bool, replicaLabels []string, maxResolutionMillis int64, partialResponse, skipChunks bool) storage.Queryable { return &queryable{ logger: logger, replicaLabels: replicaLabels, @@ -31,6 +31,7 @@ func NewQueryableCreator(logger log.Logger, proxy storepb.StoreServer) Queryable deduplicate: deduplicate, maxResolutionMillis: maxResolutionMillis, partialResponse: partialResponse, + skipChunks: skipChunks, } } } @@ -42,11 +43,12 @@ type queryable struct { deduplicate bool maxResolutionMillis int64 partialResponse bool + skipChunks bool } // Querier returns a new storage querier against the underlying proxy store API. func (q *queryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - return newQuerier(ctx, q.logger, mint, maxt, q.replicaLabels, q.proxy, q.deduplicate, int64(q.maxResolutionMillis), q.partialResponse), nil + return newQuerier(ctx, q.logger, mint, maxt, q.replicaLabels, q.proxy, q.deduplicate, int64(q.maxResolutionMillis), q.partialResponse, q.skipChunks), nil } type querier struct { @@ -59,6 +61,7 @@ type querier struct { deduplicate bool maxResolutionMillis int64 partialResponse bool + skipChunks bool } // newQuerier creates implementation of storage.Querier that fetches data from the proxy @@ -72,6 +75,7 @@ func newQuerier( deduplicate bool, maxResolutionMillis int64, partialResponse bool, + skipChunks bool, ) *querier { if logger == nil { logger = log.NewNopLogger() @@ -93,6 +97,7 @@ func newQuerier( deduplicate: deduplicate, maxResolutionMillis: maxResolutionMillis, partialResponse: partialResponse, + skipChunks: skipChunks, } } @@ -185,6 +190,7 @@ func (q *querier) Select(params *storage.SelectParams, ms ...*labels.Matcher) (s MaxResolutionWindow: q.maxResolutionMillis, Aggregates: queryAggrs, PartialResponseDisabled: !q.partialResponse, + SkipChunks: q.skipChunks, }, resp); err != nil { return nil, nil, errors.Wrap(err, "proxy Series()") } diff --git a/pkg/query/querier_test.go b/pkg/query/querier_test.go index bf84218c3e..57ea33bea9 100644 --- a/pkg/query/querier_test.go +++ b/pkg/query/querier_test.go @@ -28,7 +28,7 @@ func TestQueryableCreator_MaxResolution(t *testing.T) { queryableCreator := NewQueryableCreator(nil, testProxy) oneHourMillis := int64(1*time.Hour) / int64(time.Millisecond) - queryable := queryableCreator(false, nil, oneHourMillis, false) + queryable := queryableCreator(false, nil, oneHourMillis, false, false) q, err := queryable.Querier(context.Background(), 0, 42) testutil.Ok(t, err) @@ -55,7 +55,7 @@ func TestQuerier_DownsampledData(t *testing.T) { }, } - q := NewQueryableCreator(nil, testProxy)(false, nil, 9999999, false) + q := NewQueryableCreator(nil, testProxy)(false, nil, 9999999, false, false) engine := promql.NewEngine( promql.EngineOpts{ @@ -176,7 +176,7 @@ func TestQuerier_Series(t *testing.T) { // Querier clamps the range to [1,300], which should drop some samples of the result above. // The store API allows endpoints to send more data then initially requested. - q := newQuerier(context.Background(), nil, 1, 300, []string{""}, testProxy, false, 0, true) + q := newQuerier(context.Background(), nil, 1, 300, []string{""}, testProxy, false, 0, true, false) defer func() { testutil.Ok(t, q.Close()) }() res, _, err := q.Select(&storage.SelectParams{}) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index ddb9edb114..6926da769e 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -946,11 +946,16 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie for set.Next() { var series storepb.Series - series.Labels, series.Chunks = set.At() - stats.mergedSeriesCount++ - stats.mergedChunksCount += len(series.Chunks) - s.metrics.chunkSizeBytes.Observe(float64(chunksSize(series.Chunks))) + + if req.SkipChunks { + series.Labels, _ = set.At() + } else { + series.Labels, series.Chunks = set.At() + + stats.mergedChunksCount += len(series.Chunks) + s.metrics.chunkSizeBytes.Observe(float64(chunksSize(series.Chunks))) + } if err := srv.Send(storepb.NewSeriesResponse(&series)); err != nil { return status.Error(codes.Unknown, errors.Wrap(err, "send series response").Error()) diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index d860887b3a..d7623e0f84 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -173,8 +173,9 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { // TODO(bwplotka): Add those test cases to TSDB querier_test.go as well, there are no tests for matching. for i, tcase := range []struct { - req *storepb.SeriesRequest - expected [][]storepb.Label + req *storepb.SeriesRequest + expected [][]storepb.Label + expectedChunkLen int }{ { req: &storepb.SeriesRequest{ @@ -184,6 +185,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MinTime: mint, MaxTime: maxt, }, + expectedChunkLen: 3, expected: [][]storepb.Label{ {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, @@ -203,6 +205,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MinTime: mint, MaxTime: maxt, }, + expectedChunkLen: 3, expected: [][]storepb.Label{ {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, @@ -218,6 +221,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MinTime: mint, MaxTime: maxt, }, + expectedChunkLen: 3, expected: [][]storepb.Label{ {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, @@ -233,6 +237,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MinTime: mint, MaxTime: maxt, }, + expectedChunkLen: 3, expected: [][]storepb.Label{ {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, @@ -252,6 +257,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MinTime: mint, MaxTime: maxt, }, + expectedChunkLen: 3, expected: [][]storepb.Label{ {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, @@ -271,6 +277,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MinTime: mint, MaxTime: maxt, }, + expectedChunkLen: 3, expected: [][]storepb.Label{ {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, {{Name: "a", Value: "2"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, @@ -286,6 +293,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MinTime: mint, MaxTime: maxt, }, + expectedChunkLen: 3, expected: [][]storepb.Label{ {{Name: "a", Value: "1"}, {Name: "c", Value: "1"}, {Name: "ext2", Value: "value2"}}, {{Name: "a", Value: "1"}, {Name: "c", Value: "2"}, {Name: "ext2", Value: "value2"}}, @@ -309,6 +317,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MinTime: mint, MaxTime: maxt, }, + expectedChunkLen: 3, expected: [][]storepb.Label{ {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, @@ -324,6 +333,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MinTime: mint, MaxTime: maxt, }, + expectedChunkLen: 3, expected: [][]storepb.Label{ {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, @@ -347,6 +357,24 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { MaxTime: maxt, }, }, + // Test no-chunk option. + { + req: &storepb.SeriesRequest{ + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "1"}, + }, + MinTime: mint, + MaxTime: maxt, + SkipChunks: true, + }, + expectedChunkLen: 0, + expected: [][]storepb.Label{ + {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, + {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, + {{Name: "a", Value: "1"}, {Name: "c", Value: "1"}, {Name: "ext2", Value: "value2"}}, + {{Name: "a", Value: "1"}, {Name: "c", Value: "2"}, {Name: "ext2", Value: "value2"}}, + }, + }, } { t.Log("Run ", i) @@ -357,7 +385,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { for i, s := range srv.SeriesSet { testutil.Equals(t, tcase.expected[i], s.Labels) - testutil.Equals(t, 3, len(s.Chunks)) + testutil.Equals(t, tcase.expectedChunkLen, len(s.Chunks)) } } } diff --git a/pkg/store/matchers.go b/pkg/store/matchers.go index d9e4c4065b..66902f5116 100644 --- a/pkg/store/matchers.go +++ b/pkg/store/matchers.go @@ -33,3 +33,21 @@ func translateMatchers(ms []storepb.LabelMatcher) (res []*labels.Matcher, err er } return res, nil } + +// matchersToString converts label matchers to string format. +func matchersToString(ms []storepb.LabelMatcher) (string, error) { + var res string + matchers, err := translateMatchers(ms) + if err != nil { + return "", err + } + + for i, m := range matchers { + res += m.String() + if i < len(matchers)-1 { + res += ", " + } + } + + return "{" + res + "}", nil +} diff --git a/pkg/store/matchers_test.go b/pkg/store/matchers_test.go new file mode 100644 index 0000000000..75caf1791b --- /dev/null +++ b/pkg/store/matchers_test.go @@ -0,0 +1,86 @@ +package store + +import ( + "testing" + + "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestMatchersToString(t *testing.T) { + cases := []struct { + ms []storepb.LabelMatcher + expected string + }{ + { + ms: []storepb.LabelMatcher{ + { + Name: "__name__", + Type: storepb.LabelMatcher_EQ, + Value: "up", + }}, + expected: `{__name__="up"}`, + }, + { + ms: []storepb.LabelMatcher{ + { + Name: "__name__", + Type: storepb.LabelMatcher_NEQ, + Value: "up", + }, + { + Name: "job", + Type: storepb.LabelMatcher_EQ, + Value: "test", + }, + }, + expected: `{__name__!="up", job="test"}`, + }, + { + ms: []storepb.LabelMatcher{ + { + Name: "__name__", + Type: storepb.LabelMatcher_EQ, + Value: "up", + }, + { + Name: "job", + Type: storepb.LabelMatcher_RE, + Value: "test", + }, + }, + expected: `{__name__="up", job=~"test"}`, + }, + { + ms: []storepb.LabelMatcher{ + { + Name: "job", + Type: storepb.LabelMatcher_NRE, + Value: "test", + }}, + expected: `{job!~"test"}`, + }, + { + ms: []storepb.LabelMatcher{ + { + Name: "__name__", + Type: storepb.LabelMatcher_EQ, + Value: "up", + }, + { + Name: "__name__", + Type: storepb.LabelMatcher_NEQ, + Value: "up", + }, + }, + // We cannot use up{__name__!="up"} in this case. + expected: `{__name__="up", __name__!="up"}`, + }, + } + + for i, c := range cases { + actual, err := matchersToString(c.ms) + testutil.Ok(t, err) + testutil.Assert(t, actual == c.expected, "test case %d failed, expected %s, actual %s", i, c.expected, actual) + } +} diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 442dc7b770..4012eceb9c 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -19,7 +19,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/gogo/protobuf/proto" "github.com/golang/snappy" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/common/version" "github.com/prometheus/prometheus/pkg/labels" @@ -59,6 +59,8 @@ type PrometheusStore struct { const ( initialBufSize = 32 * 1024 // 32KB seems like a good minimum starting size. + + SUCCESS = "success" ) // NewPrometheusStore returns a new PrometheusStore that uses the given HTTP client @@ -150,7 +152,7 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, s storepb.Store_Serie } if len(newMatchers) == 0 { - return status.Error(codes.InvalidArgument, errors.New("no matchers specified (excluding external labels)").Error()) + return status.Error(codes.InvalidArgument, "no matchers specified (excluding external labels)") } // Don't ask for more than available time. This includes potential `minTime` flag limit. @@ -159,6 +161,32 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, s storepb.Store_Serie r.MinTime = availableMinTime } + if r.SkipChunks { + labelMaps, err := p.seriesLabels(s.Context(), newMatchers, r.MinTime, r.MaxTime) + if err != nil { + return err + } + for _, lbm := range labelMaps { + lset := make([]storepb.Label, 0, len(lbm)+len(externalLabels)) + for k, v := range lbm { + lset = append(lset, storepb.Label{Name: k, Value: v}) + } + for _, l := range externalLabels { + lset = append(lset, storepb.Label{ + Name: l.Name, + Value: l.Value, + }) + } + sort.Slice(lset, func(i, j int) bool { + return lset[i].Name < lset[j].Name + }) + if err = s.Send(storepb.NewSeriesResponse(&storepb.Series{Labels: lset})); err != nil { + return err + } + } + return nil + } + q := &prompb.Query{StartTimestampMs: r.MinTime, EndTimestampMs: r.MaxTime} for _, m := range newMatchers { @@ -427,7 +455,7 @@ func (p *PrometheusStore) startPromSeries(ctx context.Context, q *prompb.Query) return presp, nil } -// matchesExternalLabels filters out external labels matching from matcher if exsits as the local storage does not have them. +// matchesExternalLabels filters out external labels matching from matcher if exists as the local storage does not have them. // It also returns false if given matchers are not matching external labels. func matchesExternalLabels(ms []storepb.LabelMatcher, externalLabels labels.Labels) (bool, []storepb.LabelMatcher, error) { if len(externalLabels) == 0 { @@ -474,7 +502,7 @@ func (p *PrometheusStore) encodeChunk(ss []prompb.Sample) (storepb.Chunk_Encodin } // translateAndExtendLabels transforms a metrics into a protobuf label set. It additionally -// attaches the given labels to it, overwriting existing ones on colllision. +// attaches the given labels to it, overwriting existing ones on collision. func (p *PrometheusStore) translateAndExtendLabels(m []prompb.Label, extend labels.Labels) []storepb.Label { lset := make([]storepb.Label, 0, len(m)+len(extend)) @@ -527,7 +555,7 @@ func (p *PrometheusStore) LabelNames(ctx context.Context, _ *storepb.LabelNamesR defer runutil.ExhaustCloseWithLogOnErr(p.logger, resp.Body, "label names request body") if resp.StatusCode/100 != 2 { - return nil, status.Error(codes.Internal, fmt.Sprintf("request Prometheus server failed, code %s", resp.Status)) + return nil, status.Errorf(codes.Internal, "request Prometheus server failed, code %s", resp.Status) } if resp.StatusCode == http.StatusNoContent { @@ -549,7 +577,7 @@ func (p *PrometheusStore) LabelNames(ctx context.Context, _ *storepb.LabelNamesR return nil, status.Error(codes.Internal, err.Error()) } - if m.Status != "success" { + if m.Status != SUCCESS { code, exists := statusToCode[resp.StatusCode] if !exists { return nil, status.Error(codes.Internal, m.Error) @@ -588,7 +616,7 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue defer runutil.ExhaustCloseWithLogOnErr(p.logger, resp.Body, "label values request body") if resp.StatusCode/100 != 2 { - return nil, status.Error(codes.Internal, fmt.Sprintf("request Prometheus server failed, code %s", resp.Status)) + return nil, status.Errorf(codes.Internal, "request Prometheus server failed, code %s", resp.Status) } if resp.StatusCode == http.StatusNoContent { @@ -611,7 +639,7 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue sort.Strings(m.Data) - if m.Status != "success" { + if m.Status != SUCCESS { code, exists := statusToCode[resp.StatusCode] if !exists { return nil, status.Error(codes.Internal, m.Error) @@ -621,3 +649,68 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue return &storepb.LabelValuesResponse{Values: m.Data}, nil } + +// seriesLabels returns the labels from Prometheus series API. +func (p *PrometheusStore) seriesLabels(ctx context.Context, matchers []storepb.LabelMatcher, startTime, endTime int64) ([]map[string]string, error) { + u := *p.base + u.Path = path.Join(u.Path, "/api/v1/series") + q := u.Query() + + metric, err := matchersToString(matchers) + if err != nil { + return nil, errors.Wrap(err, "invalid matchers") + } + + q.Add("match[]", metric) + u.RawQuery = q.Encode() + q.Add("start", string(startTime)) + q.Add("end", string(endTime)) + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + req.Header.Set("User-Agent", userAgent) + + span, ctx := tracing.StartSpan(ctx, "/prom_series HTTP[client]") + defer span.Finish() + + resp, err := p.client.Do(req.WithContext(ctx)) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer runutil.ExhaustCloseWithLogOnErr(p.logger, resp.Body, "series request body") + + if resp.StatusCode/100 != 2 { + return nil, status.Errorf(codes.Internal, "request Prometheus server failed, code %s", resp.Status) + } + + if resp.StatusCode == http.StatusNoContent { + return nil, nil + } + + var m struct { + Data []map[string]string `json:"data"` + Status string `json:"status"` + Error string `json:"error"` + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + if err = json.Unmarshal(body, &m); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + if m.Status != SUCCESS { + code, exists := statusToCode[resp.StatusCode] + if !exists { + return nil, status.Error(codes.Internal, m.Error) + } + return nil, status.Error(code, m.Error) + } + + return m.Data, nil +} diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index bb44ab12d7..5db3ee75cc 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -139,6 +139,26 @@ func testPrometheusStoreSeriesE2e(t *testing.T, prefix string) { testutil.NotOk(t, err) testutil.Equals(t, "rpc error: code = InvalidArgument desc = no matchers specified (excluding external labels)", err.Error()) } + // Set no-chunk option to only get the series labels. + { + srv := newStoreSeriesServer(ctx) + testutil.Ok(t, proxy.Series(&storepb.SeriesRequest{ + MinTime: 0, + MaxTime: baseT + 300, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "b"}, + }, + SkipChunks: true, + }, srv)) + testutil.Equals(t, 1, len(srv.SeriesSet)) + + testutil.Equals(t, []storepb.Label{ + {Name: "a", Value: "b"}, + {Name: "region", Value: "eu-west"}, + }, srv.SeriesSet[0].Labels) + + testutil.Equals(t, 0, len(srv.SeriesSet[0].Chunks)) + } } type sample struct { @@ -160,6 +180,86 @@ func getExternalLabels() labels.Labels { {Name: "ext_b", Value: "a"}} } +func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { + t.Helper() + + defer leaktest.CheckTimeout(t, 10*time.Second)() + + p, err := testutil.NewPrometheus() + testutil.Ok(t, err) + defer func() { testutil.Ok(t, p.Stop()) }() + + baseT := timestamp.FromTime(time.Now()) / 1000 * 1000 + + a := p.Appender() + _, err = a.Add(labels.FromStrings("a", "b"), baseT+100, 1) + testutil.Ok(t, err) + _, err = a.Add(labels.FromStrings("a", "c", "job", "test"), baseT+200, 2) + testutil.Ok(t, err) + _, err = a.Add(labels.FromStrings("a", "d", "job", "test"), baseT+300, 3) + testutil.Ok(t, err) + testutil.Ok(t, a.Commit()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + testutil.Ok(t, p.Start()) + + u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) + testutil.Ok(t, err) + + limitMinT := int64(0) + proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, + func() labels.Labels { return labels.FromStrings("region", "eu-west") }, + func() (int64, int64) { return limitMinT, -1 }) // Maxt does not matter. + testutil.Ok(t, err) + + { + res, err := proxy.seriesLabels(ctx, []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "b"}, + }, baseT, baseT+300) + testutil.Ok(t, err) + + testutil.Equals(t, len(res), 1) + testutil.Equals(t, labels.FromMap(res[0]), labels.Labels{{Name: "a", Value: "b"}}) + } + { + res, err := proxy.seriesLabels(ctx, []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "foo"}, + }, baseT, baseT+300) + testutil.Ok(t, err) + + testutil.Equals(t, len(res), 0) + } + { + res, err := proxy.seriesLabels(ctx, []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_NEQ, Name: "a", Value: "b"}, + {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "test"}, + }, baseT, baseT+300) + testutil.Ok(t, err) + + // Get two metrics {a="c", job="test"} and {a="d", job="test"}. + testutil.Equals(t, len(res), 2) + for _, r := range res { + testutil.Equals(t, labels.FromMap(r).Has("a"), true) + testutil.Equals(t, labels.FromMap(r).Get("job"), "test") + } + } + { + res, err := proxy.seriesLabels(ctx, []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "test"}, + }, baseT, baseT+300) + testutil.Ok(t, err) + + // Get two metrics. + testutil.Equals(t, len(res), 2) + for _, r := range res { + testutil.Equals(t, labels.FromMap(r).Has("a"), true) + testutil.Equals(t, labels.FromMap(r).Get("job"), "test") + } + } +} + func TestPrometheusStore_LabelValues_e2e(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 1817007da7..574c5e0e4b 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -211,6 +211,7 @@ func (s *ProxyStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSe Matchers: newMatchers, Aggregates: r.Aggregates, MaxResolutionWindow: r.MaxResolutionWindow, + SkipChunks: r.SkipChunks, PartialResponseDisabled: r.PartialResponseDisabled, } wg = &sync.WaitGroup{} diff --git a/pkg/store/proxy_test.go b/pkg/store/proxy_test.go index cc006a4a8e..d19bf58f9a 100644 --- a/pkg/store/proxy_test.go +++ b/pkg/store/proxy_test.go @@ -408,6 +408,32 @@ func TestProxyStore_Series(t *testing.T) { }, expectedErr: errors.New("fetch series for [name:\"ext\" value:\"1\" ] test: error!"), }, + { + title: "use no chunk to only get labels", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storeSeriesResponse(t, labels.FromStrings("a", "a")), + }, + }, + minTime: 1, + maxTime: 300, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + SkipChunks: true, + }, + expectedSeries: []rawSeries{ + { + lset: []storepb.Label{{Name: "a", Value: "a"}}, + }, + }, + }, } { if ok := t.Run(tc.title, func(t *testing.T) { diff --git a/pkg/store/storepb/rpc.pb.go b/pkg/store/storepb/rpc.pb.go index a4a34bd29a..77b5b5fa03 100644 --- a/pkg/store/storepb/rpc.pb.go +++ b/pkg/store/storepb/rpc.pb.go @@ -26,7 +26,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type StoreType int32 @@ -262,6 +262,8 @@ type SeriesRequest struct { PartialResponseDisabled bool `protobuf:"varint,6,opt,name=partial_response_disabled,json=partialResponseDisabled,proto3" json:"partial_response_disabled,omitempty"` // TODO(bwplotka): Move Thanos components to use strategy instead. Including QueryAPI. PartialResponseStrategy PartialResponseStrategy `protobuf:"varint,7,opt,name=partial_response_strategy,json=partialResponseStrategy,proto3,enum=thanos.PartialResponseStrategy" json:"partial_response_strategy,omitempty"` + // skip_chunks controls whether sending chunks or not in series responses. + SkipChunks bool `protobuf:"varint,8,opt,name=skip_chunks,json=skipChunks,proto3" json:"skip_chunks,omitempty"` } func (m *SeriesRequest) Reset() { *m = SeriesRequest{} } @@ -344,10 +346,10 @@ type isSeriesResponse_Result interface { } type SeriesResponse_Series struct { - Series *Series `protobuf:"bytes,1,opt,name=series,proto3,oneof"` + Series *Series `protobuf:"bytes,1,opt,name=series,proto3,oneof" json:"series,omitempty"` } type SeriesResponse_Warning struct { - Warning string `protobuf:"bytes,2,opt,name=warning,proto3,oneof"` + Warning string `protobuf:"bytes,2,opt,name=warning,proto3,oneof" json:"warning,omitempty"` } func (*SeriesResponse_Series) isSeriesResponse_Result() {} @@ -374,76 +376,14 @@ func (m *SeriesResponse) GetWarning() string { return "" } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*SeriesResponse) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _SeriesResponse_OneofMarshaler, _SeriesResponse_OneofUnmarshaler, _SeriesResponse_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*SeriesResponse) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*SeriesResponse_Series)(nil), (*SeriesResponse_Warning)(nil), } } -func _SeriesResponse_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*SeriesResponse) - // result - switch x := m.Result.(type) { - case *SeriesResponse_Series: - _ = b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Series); err != nil { - return err - } - case *SeriesResponse_Warning: - _ = b.EncodeVarint(2<<3 | proto.WireBytes) - _ = b.EncodeStringBytes(x.Warning) - case nil: - default: - return fmt.Errorf("SeriesResponse.Result has unexpected type %T", x) - } - return nil -} - -func _SeriesResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*SeriesResponse) - switch tag { - case 1: // result.series - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(Series) - err := b.DecodeMessage(msg) - m.Result = &SeriesResponse_Series{msg} - return true, err - case 2: // result.warning - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Result = &SeriesResponse_Warning{x} - return true, err - default: - return false, nil - } -} - -func _SeriesResponse_OneofSizer(msg proto.Message) (n int) { - m := msg.(*SeriesResponse) - // result - switch x := m.Result.(type) { - case *SeriesResponse_Series: - s := proto.Size(x.Series) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *SeriesResponse_Warning: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.Warning))) - n += len(x.Warning) - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - type LabelNamesRequest struct { PartialResponseDisabled bool `protobuf:"varint,1,opt,name=partial_response_disabled,json=partialResponseDisabled,proto3" json:"partial_response_disabled,omitempty"` // TODO(bwplotka): Move Thanos components to use strategy instead. Including QueryAPI. @@ -617,57 +557,58 @@ func init() { func init() { proto.RegisterFile("rpc.proto", fileDescriptor_77a6da22d6a3feb1) } var fileDescriptor_77a6da22d6a3feb1 = []byte{ - // 791 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xcd, 0x6e, 0xeb, 0x44, - 0x14, 0xf6, 0xd8, 0x89, 0x13, 0x9f, 0xdc, 0x46, 0xbe, 0xd3, 0xdc, 0x7b, 0x5d, 0x23, 0xe5, 0x46, - 0x91, 0x90, 0xa2, 0x82, 0x72, 0x21, 0x08, 0x10, 0xec, 0x92, 0xd4, 0x55, 0x23, 0xda, 0x04, 0x26, - 0x49, 0xc3, 0xcf, 0x22, 0x38, 0xed, 0xe0, 0x5a, 0x72, 0xec, 0xe0, 0x71, 0x68, 0xbb, 0xe5, 0x09, - 0x78, 0x10, 0xde, 0x82, 0x4d, 0x97, 0x5d, 0xc2, 0x06, 0x41, 0xfb, 0x12, 0x2c, 0x91, 0xc7, 0xe3, - 0x24, 0x86, 0xb6, 0xd2, 0x55, 0x76, 0x73, 0xbe, 0xef, 0xf8, 0x9c, 0xf9, 0xbe, 0x39, 0x33, 0x06, - 0x2d, 0x5c, 0x9c, 0x35, 0x17, 0x61, 0x10, 0x05, 0x58, 0x8d, 0x2e, 0x6c, 0x3f, 0x60, 0x66, 0x29, - 0xba, 0x5e, 0x50, 0x96, 0x80, 0x66, 0xc5, 0x09, 0x9c, 0x80, 0x2f, 0xdf, 0xc4, 0xab, 0x04, 0xad, - 0xef, 0x40, 0xa9, 0xe7, 0xff, 0x10, 0x10, 0xfa, 0xe3, 0x92, 0xb2, 0xa8, 0xfe, 0x07, 0x82, 0x67, - 0x49, 0xcc, 0x16, 0x81, 0xcf, 0x28, 0x7e, 0x0f, 0x54, 0xcf, 0x9e, 0x51, 0x8f, 0x19, 0xa8, 0xa6, - 0x34, 0x4a, 0xad, 0x9d, 0x66, 0x52, 0xbb, 0x79, 0x1c, 0xa3, 0x9d, 0xdc, 0xcd, 0x9f, 0xaf, 0x25, - 0x22, 0x52, 0xf0, 0x1e, 0x14, 0xe7, 0xae, 0x3f, 0x8d, 0xdc, 0x39, 0x35, 0xe4, 0x1a, 0x6a, 0x28, - 0xa4, 0x30, 0x77, 0xfd, 0x91, 0x3b, 0xa7, 0x9c, 0xb2, 0xaf, 0x12, 0x4a, 0x11, 0x94, 0x7d, 0xc5, - 0xa9, 0x37, 0xa0, 0xb1, 0x28, 0x08, 0xe9, 0xe8, 0x7a, 0x41, 0x8d, 0x5c, 0x0d, 0x35, 0xca, 0xad, - 0xe7, 0x69, 0x97, 0x61, 0x4a, 0x90, 0x75, 0x0e, 0xfe, 0x18, 0x80, 0x37, 0x9c, 0x32, 0x1a, 0x31, - 0x23, 0xcf, 0xf7, 0xa5, 0x67, 0xf6, 0x35, 0xa4, 0x91, 0xd8, 0x9a, 0xe6, 0x89, 0x98, 0xd5, 0x3f, - 0x85, 0x62, 0x4a, 0xbe, 0x95, 0xac, 0xfa, 0x3f, 0x32, 0xec, 0x0c, 0x69, 0xe8, 0x52, 0x26, 0x6c, - 0xca, 0x08, 0x45, 0x8f, 0x0b, 0x95, 0xb3, 0x42, 0x3f, 0x89, 0xa9, 0xe8, 0xec, 0x82, 0x86, 0xcc, - 0x50, 0x78, 0xdb, 0x4a, 0xa6, 0xed, 0x49, 0x42, 0x8a, 0xee, 0xab, 0x5c, 0xdc, 0x82, 0x17, 0x71, - 0xc9, 0x90, 0xb2, 0xc0, 0x5b, 0x46, 0x6e, 0xe0, 0x4f, 0x2f, 0x5d, 0xff, 0x3c, 0xb8, 0xe4, 0x66, - 0x29, 0x64, 0x77, 0x6e, 0x5f, 0x91, 0x15, 0x37, 0xe1, 0x14, 0x7e, 0x1f, 0xc0, 0x76, 0x9c, 0x90, - 0x3a, 0x76, 0x44, 0x13, 0x8f, 0xca, 0xad, 0x67, 0x69, 0xb7, 0xb6, 0xe3, 0x84, 0x64, 0x83, 0xc7, - 0x9f, 0xc3, 0xde, 0xc2, 0x0e, 0x23, 0xd7, 0xf6, 0xe2, 0x2e, 0xfc, 0xe4, 0xa7, 0xe7, 0x2e, 0xb3, - 0x67, 0x1e, 0x3d, 0x37, 0xd4, 0x1a, 0x6a, 0x14, 0xc9, 0x2b, 0x91, 0x90, 0x4e, 0xc6, 0x81, 0xa0, - 0xf1, 0x77, 0x0f, 0x7c, 0xcb, 0xa2, 0xd0, 0x8e, 0xa8, 0x73, 0x6d, 0x14, 0xf8, 0x71, 0xbe, 0x4e, - 0x1b, 0x7f, 0x99, 0xad, 0x31, 0x14, 0x69, 0xff, 0x2b, 0x9e, 0x12, 0xf5, 0xef, 0xa1, 0x9c, 0x3a, - 0x2f, 0x06, 0xb2, 0x01, 0x2a, 0xe3, 0x08, 0x37, 0xbe, 0xd4, 0x2a, 0xaf, 0x46, 0x85, 0xa3, 0x47, - 0x12, 0x11, 0x3c, 0x36, 0xa1, 0x70, 0x69, 0x87, 0xbe, 0xeb, 0x3b, 0xfc, 0x20, 0xb4, 0x23, 0x89, - 0xa4, 0x40, 0xa7, 0x08, 0x6a, 0x48, 0xd9, 0xd2, 0x8b, 0xea, 0xbf, 0x22, 0x78, 0xce, 0xdd, 0xef, - 0xdb, 0xf3, 0xf5, 0x01, 0x3f, 0x69, 0x08, 0xda, 0xc2, 0x10, 0x79, 0x4b, 0x43, 0x0e, 0x01, 0x6f, - 0xee, 0x56, 0x98, 0x52, 0x81, 0xbc, 0x1f, 0x03, 0x7c, 0x9a, 0x35, 0x92, 0x04, 0xd8, 0x84, 0xa2, - 0xd0, 0xcb, 0x0c, 0x99, 0x13, 0xab, 0xb8, 0xfe, 0x1b, 0x12, 0x85, 0x4e, 0x6d, 0x6f, 0xb9, 0xd6, - 0x5d, 0x81, 0x3c, 0x1f, 0x7a, 0xae, 0x51, 0x23, 0x49, 0xf0, 0xb4, 0x1b, 0xf2, 0x16, 0x6e, 0x28, - 0x5b, 0xba, 0xd1, 0x83, 0xdd, 0x8c, 0x08, 0x61, 0xc7, 0x4b, 0x50, 0x7f, 0xe2, 0x88, 0xf0, 0x43, - 0x44, 0x4f, 0x19, 0xb2, 0x4f, 0x40, 0x5b, 0x3d, 0x36, 0xb8, 0x04, 0x85, 0x71, 0xff, 0x8b, 0xfe, - 0x60, 0xd2, 0xd7, 0x25, 0xac, 0x41, 0xfe, 0xab, 0xb1, 0x45, 0xbe, 0xd1, 0x11, 0x2e, 0x42, 0x8e, - 0x8c, 0x8f, 0x2d, 0x5d, 0x8e, 0x33, 0x86, 0xbd, 0x03, 0xab, 0xdb, 0x26, 0xba, 0x12, 0x67, 0x0c, - 0x47, 0x03, 0x62, 0xe9, 0xb9, 0x18, 0x27, 0x56, 0xd7, 0xea, 0x9d, 0x5a, 0x7a, 0x7e, 0xbf, 0x09, - 0xaf, 0x1e, 0x91, 0x14, 0x57, 0x9a, 0xb4, 0x89, 0x28, 0xdf, 0xee, 0x0c, 0xc8, 0x48, 0x47, 0xfb, - 0x1d, 0xc8, 0xc5, 0x57, 0x13, 0x17, 0x40, 0x21, 0xed, 0x49, 0xc2, 0x75, 0x07, 0xe3, 0xfe, 0x48, - 0x47, 0x31, 0x36, 0x1c, 0x9f, 0xe8, 0x72, 0xbc, 0x38, 0xe9, 0xf5, 0x75, 0x85, 0x2f, 0xda, 0x5f, - 0x27, 0x3d, 0x79, 0x96, 0x45, 0xf4, 0x7c, 0xeb, 0x67, 0x19, 0xf2, 0x5c, 0x08, 0xfe, 0x10, 0x72, - 0xf1, 0x53, 0x8e, 0x77, 0x53, 0x7b, 0x37, 0x1e, 0x7a, 0xb3, 0x92, 0x05, 0x85, 0x71, 0x9f, 0x81, - 0x9a, 0x5c, 0x23, 0xfc, 0x22, 0x7b, 0xad, 0xd2, 0xcf, 0x5e, 0xfe, 0x17, 0x4e, 0x3e, 0xfc, 0x00, - 0xe1, 0x2e, 0xc0, 0x7a, 0x30, 0xf1, 0x5e, 0xe6, 0x61, 0xdb, 0xbc, 0x5a, 0xa6, 0xf9, 0x10, 0x25, - 0xfa, 0x1f, 0x42, 0x69, 0xe3, 0x3c, 0x71, 0x36, 0x35, 0x33, 0xa9, 0xe6, 0x3b, 0x0f, 0x72, 0x49, - 0x9d, 0xce, 0xbb, 0x37, 0x7f, 0x57, 0xa5, 0x9b, 0xbb, 0x2a, 0xba, 0xbd, 0xab, 0xa2, 0xbf, 0xee, - 0xaa, 0xe8, 0x97, 0xfb, 0xaa, 0x74, 0x7b, 0x5f, 0x95, 0x7e, 0xbf, 0xaf, 0x4a, 0xdf, 0x16, 0xf8, - 0xaf, 0x64, 0x31, 0x9b, 0xa9, 0xfc, 0x1f, 0xf8, 0xd1, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x5c, - 0x24, 0xf3, 0x5b, 0x3b, 0x07, 0x00, 0x00, + // 814 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x4f, 0x6f, 0xe3, 0x44, + 0x14, 0xf7, 0xd8, 0x89, 0x13, 0xbf, 0x6c, 0x2b, 0xef, 0x34, 0xbb, 0xeb, 0x1a, 0x29, 0xad, 0x2c, + 0x21, 0x45, 0x05, 0x65, 0x21, 0x08, 0x10, 0xdc, 0x92, 0xac, 0x57, 0x1b, 0xb1, 0x4d, 0x60, 0x92, + 0x6c, 0xf8, 0x73, 0x08, 0x4e, 0x3b, 0xb8, 0xd6, 0x3a, 0xb6, 0xf1, 0x38, 0xb4, 0xbd, 0xf2, 0x09, + 0xb8, 0xf2, 0x1d, 0xf8, 0x16, 0x5c, 0x7a, 0xdc, 0x23, 0x5c, 0x10, 0xb4, 0x5f, 0x04, 0x79, 0x3c, + 0x4e, 0x62, 0x68, 0x2b, 0xad, 0x72, 0x9b, 0xf7, 0xfb, 0xbd, 0x79, 0x6f, 0xde, 0xef, 0xbd, 0x99, + 0x01, 0x2d, 0x8e, 0x4e, 0x5a, 0x51, 0x1c, 0x26, 0x21, 0x56, 0x93, 0x33, 0x27, 0x08, 0x99, 0x59, + 0x4b, 0x2e, 0x23, 0xca, 0x32, 0xd0, 0xac, 0xbb, 0xa1, 0x1b, 0xf2, 0xe5, 0xd3, 0x74, 0x95, 0xa1, + 0xd6, 0x0e, 0xd4, 0xfa, 0xc1, 0x0f, 0x21, 0xa1, 0x3f, 0x2e, 0x29, 0x4b, 0xac, 0x3f, 0x11, 0x3c, + 0xc8, 0x6c, 0x16, 0x85, 0x01, 0xa3, 0xf8, 0x3d, 0x50, 0x7d, 0x67, 0x4e, 0x7d, 0x66, 0xa0, 0x43, + 0xa5, 0x59, 0x6b, 0xef, 0xb4, 0xb2, 0xd8, 0xad, 0x97, 0x29, 0xda, 0x2d, 0x5d, 0xfd, 0x75, 0x20, + 0x11, 0xe1, 0x82, 0xf7, 0xa1, 0xba, 0xf0, 0x82, 0x59, 0xe2, 0x2d, 0xa8, 0x21, 0x1f, 0xa2, 0xa6, + 0x42, 0x2a, 0x0b, 0x2f, 0x18, 0x7b, 0x0b, 0xca, 0x29, 0xe7, 0x22, 0xa3, 0x14, 0x41, 0x39, 0x17, + 0x9c, 0x7a, 0x0a, 0x1a, 0x4b, 0xc2, 0x98, 0x8e, 0x2f, 0x23, 0x6a, 0x94, 0x0e, 0x51, 0x73, 0xb7, + 0xfd, 0x30, 0xcf, 0x32, 0xca, 0x09, 0xb2, 0xf6, 0xc1, 0x1f, 0x03, 0xf0, 0x84, 0x33, 0x46, 0x13, + 0x66, 0x94, 0xf9, 0xb9, 0xf4, 0xc2, 0xb9, 0x46, 0x34, 0x11, 0x47, 0xd3, 0x7c, 0x61, 0x33, 0xeb, + 0x53, 0xa8, 0xe6, 0xe4, 0x5b, 0x95, 0x65, 0xfd, 0xaa, 0xc0, 0xce, 0x88, 0xc6, 0x1e, 0x65, 0x42, + 0xa6, 0x42, 0xa1, 0xe8, 0xee, 0x42, 0xe5, 0x62, 0xa1, 0x9f, 0xa4, 0x54, 0x72, 0x72, 0x46, 0x63, + 0x66, 0x28, 0x3c, 0x6d, 0xbd, 0x90, 0xf6, 0x38, 0x23, 0x45, 0xf6, 0x95, 0x2f, 0x6e, 0xc3, 0xa3, + 0x34, 0x64, 0x4c, 0x59, 0xe8, 0x2f, 0x13, 0x2f, 0x0c, 0x66, 0xe7, 0x5e, 0x70, 0x1a, 0x9e, 0x73, + 0xb1, 0x14, 0xb2, 0xb7, 0x70, 0x2e, 0xc8, 0x8a, 0x9b, 0x72, 0x0a, 0xbf, 0x0f, 0xe0, 0xb8, 0x6e, + 0x4c, 0x5d, 0x27, 0xa1, 0x99, 0x46, 0xbb, 0xed, 0x07, 0x79, 0xb6, 0x8e, 0xeb, 0xc6, 0x64, 0x83, + 0xc7, 0x9f, 0xc3, 0x7e, 0xe4, 0xc4, 0x89, 0xe7, 0xf8, 0x69, 0x16, 0xde, 0xf9, 0xd9, 0xa9, 0xc7, + 0x9c, 0xb9, 0x4f, 0x4f, 0x0d, 0xf5, 0x10, 0x35, 0xab, 0xe4, 0x89, 0x70, 0xc8, 0x27, 0xe3, 0x99, + 0xa0, 0xf1, 0x77, 0xb7, 0xec, 0x65, 0x49, 0xec, 0x24, 0xd4, 0xbd, 0x34, 0x2a, 0xbc, 0x9d, 0x07, + 0x79, 0xe2, 0x2f, 0x8b, 0x31, 0x46, 0xc2, 0xed, 0x7f, 0xc1, 0x73, 0x02, 0x1f, 0x40, 0x8d, 0xbd, + 0xf6, 0xa2, 0xd9, 0xc9, 0xd9, 0x32, 0x78, 0xcd, 0x8c, 0x2a, 0x3f, 0x0a, 0xa4, 0x50, 0x8f, 0x23, + 0xd6, 0xf7, 0xb0, 0x9b, 0xb7, 0x46, 0x4c, 0x6c, 0x13, 0x54, 0xc6, 0x11, 0xde, 0x99, 0x5a, 0x7b, + 0x77, 0x35, 0x4b, 0x1c, 0x7d, 0x21, 0x11, 0xc1, 0x63, 0x13, 0x2a, 0xe7, 0x4e, 0x1c, 0x78, 0x81, + 0xcb, 0x3b, 0xa5, 0xbd, 0x90, 0x48, 0x0e, 0x74, 0xab, 0xa0, 0xc6, 0x94, 0x2d, 0xfd, 0xc4, 0xfa, + 0x0d, 0xc1, 0x43, 0xde, 0x9e, 0x81, 0xb3, 0x58, 0x4f, 0xc0, 0xbd, 0x8a, 0xa1, 0x2d, 0x14, 0x93, + 0xb7, 0x53, 0xcc, 0x7a, 0x0e, 0x78, 0xf3, 0xb4, 0x42, 0x94, 0x3a, 0x94, 0x83, 0x14, 0xe0, 0xe3, + 0xae, 0x91, 0xcc, 0xc0, 0x26, 0x54, 0x45, 0xbd, 0xcc, 0x90, 0x39, 0xb1, 0xb2, 0xad, 0xdf, 0x91, + 0x08, 0xf4, 0xca, 0xf1, 0x97, 0xeb, 0xba, 0xeb, 0x50, 0xe6, 0xb7, 0x82, 0xd7, 0xa8, 0x91, 0xcc, + 0xb8, 0x5f, 0x0d, 0x79, 0x0b, 0x35, 0x94, 0x2d, 0xd5, 0xe8, 0xc3, 0x5e, 0xa1, 0x08, 0x21, 0xc7, + 0x63, 0x50, 0x7f, 0xe2, 0x88, 0xd0, 0x43, 0x58, 0xf7, 0x09, 0x72, 0x44, 0x40, 0x5b, 0xbd, 0x46, + 0xb8, 0x06, 0x95, 0xc9, 0xe0, 0x8b, 0xc1, 0x70, 0x3a, 0xd0, 0x25, 0xac, 0x41, 0xf9, 0xab, 0x89, + 0x4d, 0xbe, 0xd1, 0x11, 0xae, 0x42, 0x89, 0x4c, 0x5e, 0xda, 0xba, 0x9c, 0x7a, 0x8c, 0xfa, 0xcf, + 0xec, 0x5e, 0x87, 0xe8, 0x4a, 0xea, 0x31, 0x1a, 0x0f, 0x89, 0xad, 0x97, 0x52, 0x9c, 0xd8, 0x3d, + 0xbb, 0xff, 0xca, 0xd6, 0xcb, 0x47, 0x2d, 0x78, 0x72, 0x47, 0x49, 0x69, 0xa4, 0x69, 0x87, 0x88, + 0xf0, 0x9d, 0xee, 0x90, 0x8c, 0x75, 0x74, 0xd4, 0x85, 0x52, 0x7a, 0x77, 0x71, 0x05, 0x14, 0xd2, + 0x99, 0x66, 0x5c, 0x6f, 0x38, 0x19, 0x8c, 0x75, 0x94, 0x62, 0xa3, 0xc9, 0xb1, 0x2e, 0xa7, 0x8b, + 0xe3, 0xfe, 0x40, 0x57, 0xf8, 0xa2, 0xf3, 0x75, 0x96, 0x93, 0x7b, 0xd9, 0x44, 0x2f, 0xb7, 0x7f, + 0x96, 0xa1, 0xcc, 0x0b, 0xc1, 0x1f, 0x42, 0x29, 0x7d, 0xeb, 0xf1, 0x5e, 0x2e, 0xef, 0xc6, 0x4f, + 0x60, 0xd6, 0x8b, 0xa0, 0x10, 0xee, 0x33, 0x50, 0xb3, 0x6b, 0x84, 0x1f, 0x15, 0xaf, 0x55, 0xbe, + 0xed, 0xf1, 0x7f, 0xe1, 0x6c, 0xe3, 0x07, 0x08, 0xf7, 0x00, 0xd6, 0x83, 0x89, 0xf7, 0x0b, 0x2f, + 0xdf, 0xe6, 0xd5, 0x32, 0xcd, 0xdb, 0x28, 0x91, 0xff, 0x39, 0xd4, 0x36, 0xfa, 0x89, 0x8b, 0xae, + 0x85, 0x49, 0x35, 0xdf, 0xb9, 0x95, 0xcb, 0xe2, 0x74, 0xdf, 0xbd, 0xfa, 0xa7, 0x21, 0x5d, 0x5d, + 0x37, 0xd0, 0x9b, 0xeb, 0x06, 0xfa, 0xfb, 0xba, 0x81, 0x7e, 0xb9, 0x69, 0x48, 0x6f, 0x6e, 0x1a, + 0xd2, 0x1f, 0x37, 0x0d, 0xe9, 0xdb, 0x0a, 0xff, 0x6b, 0xa2, 0xf9, 0x5c, 0xe5, 0x9f, 0xe4, 0x47, + 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x34, 0x17, 0xba, 0x5c, 0x07, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1054,6 +995,16 @@ func (m *SeriesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.SkipChunks { + i-- + if m.SkipChunks { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x40 + } if m.PartialResponseStrategy != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.PartialResponseStrategy)) i-- @@ -1152,7 +1103,8 @@ func (m *SeriesResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { } func (m *SeriesResponse_Series) MarshalTo(dAtA []byte) (int, error) { - return m.MarshalToSizedBuffer(dAtA[:m.Size()]) + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *SeriesResponse_Series) MarshalToSizedBuffer(dAtA []byte) (int, error) { @@ -1172,7 +1124,8 @@ func (m *SeriesResponse_Series) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } func (m *SeriesResponse_Warning) MarshalTo(dAtA []byte) (int, error) { - return m.MarshalToSizedBuffer(dAtA[:m.Size()]) + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *SeriesResponse_Warning) MarshalToSizedBuffer(dAtA []byte) (int, error) { @@ -1448,6 +1401,9 @@ func (m *SeriesRequest) Size() (n int) { if m.PartialResponseStrategy != 0 { n += 1 + sovRpc(uint64(m.PartialResponseStrategy)) } + if m.SkipChunks { + n += 2 + } return n } @@ -2113,6 +2069,26 @@ func (m *SeriesRequest) Unmarshal(dAtA []byte) error { break } } + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SkipChunks", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.SkipChunks = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -2710,6 +2686,7 @@ func (m *LabelValuesResponse) Unmarshal(dAtA []byte) error { func skipRpc(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 + depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -2741,10 +2718,8 @@ func skipRpc(dAtA []byte) (n int, err error) { break } } - return iNdEx, nil case 1: iNdEx += 8 - return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -2765,55 +2740,30 @@ func skipRpc(dAtA []byte) (n int, err error) { return 0, ErrInvalidLengthRpc } iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthRpc - } - return iNdEx, nil case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowRpc - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipRpc(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthRpc - } - } - return iNdEx, nil + depth++ case 4: - return iNdEx, nil + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupRpc + } + depth-- case 5: iNdEx += 4 - return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } + if iNdEx < 0 { + return 0, ErrInvalidLengthRpc + } + if depth == 0 { + return iNdEx, nil + } } - panic("unreachable") + return 0, io.ErrUnexpectedEOF } var ( - ErrInvalidLengthRpc = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowRpc = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthRpc = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowRpc = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupRpc = fmt.Errorf("proto: unexpected end of group") ) diff --git a/pkg/store/storepb/rpc.proto b/pkg/store/storepb/rpc.proto index 278d41f350..219262f207 100644 --- a/pkg/store/storepb/rpc.proto +++ b/pkg/store/storepb/rpc.proto @@ -92,6 +92,9 @@ message SeriesRequest { // TODO(bwplotka): Move Thanos components to use strategy instead. Including QueryAPI. PartialResponseStrategy partial_response_strategy = 7; + + // skip_chunks controls whether sending chunks or not in series responses. + bool skip_chunks = 8; } enum Aggr { diff --git a/pkg/store/storepb/types.pb.go b/pkg/store/storepb/types.pb.go index 5a8ae93fce..89e3787d24 100644 --- a/pkg/store/storepb/types.pb.go +++ b/pkg/store/storepb/types.pb.go @@ -22,7 +22,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type Chunk_Encoding int32 @@ -1509,6 +1509,7 @@ func (m *LabelMatcher) Unmarshal(dAtA []byte) error { func skipTypes(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 + depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -1540,10 +1541,8 @@ func skipTypes(dAtA []byte) (n int, err error) { break } } - return iNdEx, nil case 1: iNdEx += 8 - return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -1564,55 +1563,30 @@ func skipTypes(dAtA []byte) (n int, err error) { return 0, ErrInvalidLengthTypes } iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthTypes - } - return iNdEx, nil case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowTypes - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipTypes(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthTypes - } - } - return iNdEx, nil + depth++ case 4: - return iNdEx, nil + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupTypes + } + depth-- case 5: iNdEx += 4 - return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } + if iNdEx < 0 { + return 0, ErrInvalidLengthTypes + } + if depth == 0 { + return iNdEx, nil + } } - panic("unreachable") + return 0, io.ErrUnexpectedEOF } var ( - ErrInvalidLengthTypes = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowTypes = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthTypes = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowTypes = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupTypes = fmt.Errorf("proto: unexpected end of group") ) diff --git a/pkg/store/tsdb.go b/pkg/store/tsdb.go index db818d67cd..40de141811 100644 --- a/pkg/store/tsdb.go +++ b/pkg/store/tsdb.go @@ -107,20 +107,23 @@ func (s *TSDBStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSer for set.Next() { series := set.At() - // TODO(fabxc): An improvement over this trivial approach would be to directly - // use the chunks provided by TSDB in the response. - // But since the sidecar has a similar approach, optimizing here has only - // limited benefit for now. - // NOTE: XOR encoding supports a max size of 2^16 - 1 samples, so we need - // to chunk all samples into groups of no more than 2^16 - 1 - // See: https://github.com/thanos-io/thanos/pull/1038. - c, err := s.encodeChunks(series.Iterator(), math.MaxUint16) - if err != nil { - return status.Errorf(codes.Internal, "encode chunk: %s", err) - } - respSeries.Labels = s.translateAndExtendLabels(series.Labels(), s.externalLabels) - respSeries.Chunks = append(respSeries.Chunks[:0], c...) + + if !r.SkipChunks { + // TODO(fabxc): An improvement over this trivial approach would be to directly + // use the chunks provided by TSDB in the response. + // But since the sidecar has a similar approach, optimizing here has only + // limited benefit for now. + // NOTE: XOR encoding supports a max size of 2^16 - 1 samples, so we need + // to chunk all samples into groups of no more than 2^16 - 1 + // See: https://github.com/thanos-io/thanos/pull/1038. + c, err := s.encodeChunks(series.Iterator(), math.MaxUint16) + if err != nil { + return status.Errorf(codes.Internal, "encode chunk: %s", err) + } + + respSeries.Chunks = append(respSeries.Chunks[:0], c...) + } if err := srv.Send(storepb.NewSeriesResponse(&respSeries)); err != nil { return status.Error(codes.Aborted, err.Error()) diff --git a/pkg/store/tsdb_test.go b/pkg/store/tsdb_test.go index 3b1483f194..6e0354df36 100644 --- a/pkg/store/tsdb_test.go +++ b/pkg/store/tsdb_test.go @@ -34,6 +34,131 @@ func TestTSDBStore_Info(t *testing.T) { testutil.Equals(t, int64(math.MaxInt64), resp.MaxTime) } +func TestTSDBStore_Series(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + db, err := testutil.NewTSDB() + defer func() { testutil.Ok(t, db.Close()) }() + testutil.Ok(t, err) + + tsdbStore := NewTSDBStore(nil, nil, db, component.Rule, labels.FromStrings("region", "eu-west")) + + appender := db.Appender() + + for i := 1; i <= 3; i++ { + _, err = appender.Add(labels.FromStrings("a", "1"), int64(i), float64(i)) + testutil.Ok(t, err) + } + err = appender.Commit() + testutil.Ok(t, err) + + for _, tc := range []struct { + title string + req *storepb.SeriesRequest + expectedSeries []rawSeries + expectedError string + }{ + { + title: "total match series", + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 3, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "1"}, + }, + }, + expectedSeries: []rawSeries{ + { + lset: []storepb.Label{{Name: "a", Value: "1"}, {Name: "region", Value: "eu-west"}}, + chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, + }, + }, + }, + { + title: "partially match time range series", + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 2, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "1"}, + }, + }, + expectedSeries: []rawSeries{ + { + lset: []storepb.Label{{Name: "a", Value: "1"}, {Name: "region", Value: "eu-west"}}, + chunks: [][]sample{{{1, 1}, {2, 2}}}, + }, + }, + }, + { + title: "dont't match time range series", + req: &storepb.SeriesRequest{ + MinTime: 4, + MaxTime: 6, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "1"}, + }, + }, + expectedSeries: []rawSeries{}, + }, + { + title: "only match external label", + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 3, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "region", Value: "eu-west"}, + }, + }, + expectedError: "rpc error: code = InvalidArgument desc = no matchers specified (excluding external labels)", + }, + { + title: "dont't match labels", + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 3, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "b", Value: "1"}, + }, + }, + expectedSeries: []rawSeries{}, + }, + { + title: "no chunk", + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 3, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "1"}, + }, + SkipChunks: true, + }, + expectedSeries: []rawSeries{ + { + lset: []storepb.Label{{Name: "a", Value: "1"}, {Name: "region", Value: "eu-west"}}, + }, + }, + }, + } { + if ok := t.Run(tc.title, func(t *testing.T) { + srv := newStoreSeriesServer(ctx) + err := tsdbStore.Series(tc.req, srv) + if len(tc.expectedError) > 0 { + testutil.NotOk(t, err) + testutil.Equals(t, tc.expectedError, err.Error()) + } else { + testutil.Ok(t, err) + seriesEquals(t, tc.expectedSeries, srv.SeriesSet) + } + }); !ok { + return + } + } +} + func TestTSDBStore_LabelNames(t *testing.T) { var err error defer leaktest.CheckTimeout(t, 10*time.Second)() From 4d1cc62ed2844f65b234c6ca7c7a5e518673a8e5 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 6 Jan 2020 12:12:26 +0000 Subject: [PATCH 139/257] Added block.MetaFetcher logic for resilient sync of meta files. (#1934) * Created transactional GaugeVec for easy atomic Gauge change. Signed-off-by: Bartek Plotka * Added single block.MetaFetcher logic for resilient sync of meta files. This is meant to replace many inconsistent meta.json syncs places in other components. Signed-off-by: Bartlomiej Plotka * Addressed comments. Signed-off-by: Bartlomiej Plotka --- CHANGELOG.md | 1 + pkg/block/block_test.go | 7 +- pkg/block/fetcher.go | 410 ++++++++++++++++++ pkg/block/fetcher_test.go | 392 +++++++++++++++++ pkg/compact/compact_e2e_test.go | 8 +- pkg/extprom/tx_gauge.go | 89 ++++ pkg/extprom/tx_gauge_test.go | 179 ++++++++ .../objtesting/acceptance_e2e_test.go | 2 +- pkg/objstore/objtesting/foreach.go | 2 +- pkg/shipper/shipper_e2e_test.go | 2 +- pkg/store/bucket_e2e_test.go | 4 +- 11 files changed, 1081 insertions(+), 15 deletions(-) create mode 100644 pkg/block/fetcher.go create mode 100644 pkg/block/fetcher_test.go create mode 100644 pkg/extprom/tx_gauge.go create mode 100644 pkg/extprom/tx_gauge_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e2a7706202..642fe2707d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.) ## Unreleased + ### Fixed - [#1856](https://github.com/thanos-io/thanos/pull/1856) Receive: close DBReadOnly after flushing to fix a memory leak. diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index 51f07d13fe..18f71c982e 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -20,8 +20,6 @@ import ( "github.com/oklog/ulid" ) -// NOTE(bplotka): For block packages we cannot use testutil, because they import block package. Consider moving simple -// testutil methods to separate package. func TestIsBlockDir(t *testing.T) { for _, tc := range []struct { input string @@ -58,10 +56,7 @@ func TestIsBlockDir(t *testing.T) { } { t.Run(tc.input, func(t *testing.T) { id, ok := IsBlockDir(tc.input) - if ok != tc.bdir { - t.Errorf("expected block dir != %v", tc.bdir) - t.FailNow() - } + testutil.Equals(t, tc.bdir, ok) if id.Compare(tc.id) != 0 { t.Errorf("expected %s got %s", tc.id, id) diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go new file mode 100644 index 0000000000..7e2d5a5423 --- /dev/null +++ b/pkg/block/fetcher.go @@ -0,0 +1,410 @@ +package block + +import ( + "context" + "encoding/json" + stderrors "errors" + "io/ioutil" + "os" + "path" + "path/filepath" + "sync" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/relabel" + tsdberrors "github.com/prometheus/prometheus/tsdb/errors" + "github.com/prometheus/prometheus/tsdb/fileutil" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/extprom" + "github.com/thanos-io/thanos/pkg/model" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/runutil" +) + +type syncMetrics struct { + syncs prometheus.Counter + syncFailures prometheus.Counter + syncDuration prometheus.Histogram + + synced *extprom.TxGaugeVec +} + +const ( + syncMetricSubSys = "blocks_meta" + + corruptedMeta = "corrupted-meta-json" + noMeta = "no-meta-json" + loadedMeta = "loaded" + failedMeta = "failed" + + // Filter's label values. + labelExcludedMeta = "label-excluded" + timeExcludedMeta = "time-excluded" + TooFreshMeta = "too-fresh" +) + +func newSyncMetrics(r prometheus.Registerer) *syncMetrics { + var m syncMetrics + + m.syncs = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: syncMetricSubSys, + Name: "syncs_total", + Help: "Total blocks metadata synchronization attempts", + }) + m.syncFailures = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: syncMetricSubSys, + Name: "sync_failures_total", + Help: "Total blocks metadata synchronization failures", + }) + m.syncDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ + Subsystem: syncMetricSubSys, + Name: "sync_duration_seconds", + Help: "Duration of the blocks metadata synchronization in seconds", + Buckets: []float64{0.01, 1, 10, 100, 1000}, + }) + m.synced = extprom.NewTxGaugeVec(prometheus.GaugeOpts{ + Subsystem: syncMetricSubSys, + Name: "synced", + Help: "Number of block metadata synced", + }, + []string{"state"}, + []string{corruptedMeta}, + []string{noMeta}, + []string{loadedMeta}, + []string{TooFreshMeta}, + []string{failedMeta}, + []string{labelExcludedMeta}, + []string{timeExcludedMeta}, + ) + if r != nil { + r.MustRegister( + m.syncs, + m.syncFailures, + m.syncDuration, + m.synced, + ) + } + return &m +} + +type MetadataFetcher interface { + Fetch(ctx context.Context) (metas map[ulid.ULID]*metadata.Meta, partial map[ulid.ULID]error, err error) +} + +type GaugeLabeled interface { + WithLabelValues(lvs ...string) prometheus.Gauge +} + +type MetaFetcherFilter func(metas map[ulid.ULID]*metadata.Meta, synced GaugeLabeled, incompleteView bool) + +// MetaFetcher is a struct that synchronizes filtered metadata of all block in the object storage with the local state. +type MetaFetcher struct { + logger log.Logger + concurrency int + bkt objstore.BucketReader + + // Optional local directory to cache meta.json files. + cacheDir string + metrics *syncMetrics + + filters []MetaFetcherFilter + + cached map[ulid.ULID]*metadata.Meta +} + +// NewMetaFetcher constructs MetaFetcher. +func NewMetaFetcher(logger log.Logger, concurrency int, bkt objstore.BucketReader, dir string, r prometheus.Registerer, filters ...MetaFetcherFilter) (*MetaFetcher, error) { + if logger == nil { + logger = log.NewNopLogger() + } + + cacheDir := "" + if dir != "" { + cacheDir = filepath.Join(dir, "meta-syncer") + if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil { + return nil, err + } + } + + return &MetaFetcher{ + logger: log.With(logger, "component", "block.MetaFetcher"), + concurrency: concurrency, + bkt: bkt, + cacheDir: cacheDir, + metrics: newSyncMetrics(r), + filters: filters, + cached: map[ulid.ULID]*metadata.Meta{}, + }, nil +} + +var ( + ErrorSyncMetaNotFound = errors.New("meta.json not found") + ErrorSyncMetaCorrupted = errors.New("meta.json corrupted") +) + +// loadMeta returns metadata from object storage or error. +// It returns `ErrorSyncMetaNotFound` and `ErrorSyncMetaCorrupted` sentinel errors in those cases. +func (s *MetaFetcher) loadMeta(ctx context.Context, id ulid.ULID) (*metadata.Meta, error) { + var ( + metaFile = path.Join(id.String(), MetaFilename) + cachedBlockDir = filepath.Join(s.cacheDir, id.String()) + ) + + // TODO(bwplotka): If that causes problems (obj store rate limits), add longer ttl to cached items. + // For 1y and 100 block sources this generates ~1.5-3k HEAD RPM. AWS handles 330k RPM per prefix. + // TODO(bwplotka): Consider filtering by consistency delay here (can't do until compactor healthyOverride work). + ok, err := s.bkt.Exists(ctx, metaFile) + if err != nil { + return nil, errors.Wrapf(err, "meta.json file exists: %v", metaFile) + } + if !ok { + return nil, ErrorSyncMetaNotFound + } + + if m, seen := s.cached[id]; seen { + return m, nil + } + + // Best effort load from local dir. + if s.cacheDir != "" { + m, err := metadata.Read(cachedBlockDir) + if err == nil { + return m, nil + } + + if !stderrors.Is(err, os.ErrNotExist) { + level.Warn(s.logger).Log("msg", "best effort read of the local meta.json failed; removing cached block dir", "dir", cachedBlockDir, "err", err) + if err := os.RemoveAll(cachedBlockDir); err != nil { + level.Warn(s.logger).Log("msg", "best effort remove of cached dir failed; ignoring", "dir", cachedBlockDir, "err", err) + } + } + } + + level.Debug(s.logger).Log("msg", "download meta", "name", metaFile) + + r, err := s.bkt.Get(ctx, metaFile) + if s.bkt.IsObjNotFoundErr(err) { + // Meta.json was deleted between bkt.Exists and here. + return nil, errors.Wrapf(ErrorSyncMetaNotFound, "%v", err) + } + if err != nil { + return nil, errors.Wrapf(err, "get meta file: %v", metaFile) + } + + defer runutil.CloseWithLogOnErr(s.logger, r, "close bkt meta get") + + metaContent, err := ioutil.ReadAll(r) + if err != nil { + return nil, errors.Wrapf(err, "read meta file: %v", metaFile) + } + + m := &metadata.Meta{} + if err := json.Unmarshal(metaContent, m); err != nil { + return nil, errors.Wrapf(ErrorSyncMetaCorrupted, "meta.json %v unmarshal: %v", metaFile, err) + } + + if m.Version != metadata.MetaVersion1 { + return nil, errors.Errorf("unexpected meta file: %s version: %d", metaFile, m.Version) + } + + // Best effort cache in local dir. + if s.cacheDir != "" { + if err := os.MkdirAll(cachedBlockDir, os.ModePerm); err != nil { + level.Warn(s.logger).Log("msg", "best effort mkdir of the meta.json block dir failed; ignoring", "dir", cachedBlockDir, "err", err) + } + + if err := metadata.Write(s.logger, cachedBlockDir, m); err != nil { + level.Warn(s.logger).Log("msg", "best effort save of the meta.json to local dir failed; ignoring", "dir", cachedBlockDir, "err", err) + } + } + + return m, nil +} + +// Fetch returns all block metas as well as partial blocks (blocks without or with corrupted meta file) from the bucket. +// It's caller responsibility to not change the returned metadata files. Maps can be modified. +// +// Returned error indicates a failure in fetching metadata. Returned meta can be assumed as correct, with some blocks missing. +func (s *MetaFetcher) Fetch(ctx context.Context) (metas map[ulid.ULID]*metadata.Meta, partial map[ulid.ULID]error, err error) { + start := time.Now() + defer func() { + s.metrics.syncDuration.Observe(time.Since(start).Seconds()) + if err != nil { + s.metrics.syncFailures.Inc() + } + }() + s.metrics.syncs.Inc() + + metas = make(map[ulid.ULID]*metadata.Meta) + partial = make(map[ulid.ULID]error) + + var ( + wg sync.WaitGroup + ch = make(chan ulid.ULID, s.concurrency) + mtx sync.Mutex + + metaErrs tsdberrors.MultiError + ) + + s.metrics.synced.ResetTx() + + for i := 0; i < s.concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + for id := range ch { + meta, err := s.loadMeta(ctx, id) + if err == nil { + mtx.Lock() + metas[id] = meta + mtx.Unlock() + continue + } + + switch errors.Cause(err) { + default: + s.metrics.synced.WithLabelValues(failedMeta).Inc() + mtx.Lock() + metaErrs.Add(err) + mtx.Unlock() + continue + case ErrorSyncMetaNotFound: + s.metrics.synced.WithLabelValues(noMeta).Inc() + case ErrorSyncMetaCorrupted: + s.metrics.synced.WithLabelValues(corruptedMeta).Inc() + } + + mtx.Lock() + partial[id] = err + mtx.Unlock() + } + }() + } + + // Workers scheduled, distribute blocks. + err = s.bkt.Iter(ctx, "", func(name string) error { + id, ok := IsBlockDir(name) + if !ok { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case ch <- id: + } + + return nil + }) + close(ch) + + wg.Wait() + if err != nil { + return nil, nil, errors.Wrap(err, "MetaFetcher: iter bucket") + } + + incompleteView := len(metaErrs) > 0 + + // Only for complete view of blocks update the cache. + if !incompleteView { + cached := make(map[ulid.ULID]*metadata.Meta, len(metas)) + for id, m := range metas { + cached[id] = m + } + s.cached = cached + + // Best effort cleanup of disk-cached metas. + if s.cacheDir != "" { + names, err := fileutil.ReadDir(s.cacheDir) + if err != nil { + level.Warn(s.logger).Log("msg", "best effort remove of not needed cached dirs failed; ignoring", "err", err) + } else { + for _, n := range names { + id, ok := IsBlockDir(n) + if !ok { + continue + } + + if _, ok := metas[id]; ok { + continue + } + + cachedBlockDir := filepath.Join(s.cacheDir, id.String()) + + // No such block loaded, remove the local dir. + if err := os.RemoveAll(cachedBlockDir); err != nil { + level.Warn(s.logger).Log("msg", "best effort remove of not needed cached dir failed; ignoring", "dir", cachedBlockDir, "err", err) + } + } + } + } + } + + for _, f := range s.filters { + // NOTE: filter can update synced metric accordingly to the reason of the exclude. + f(metas, s.metrics.synced, incompleteView) + } + + s.metrics.synced.WithLabelValues(loadedMeta).Set(float64(len(metas))) + s.metrics.synced.Submit() + + if incompleteView { + return metas, partial, errors.Wrap(metaErrs, "incomplete view") + } + + level.Info(s.logger).Log("msg", "successfully fetched block metadata", "duration", time.Since(start).String(), "cached", len(s.cached), "returned", len(metas), "partial", len(partial)) + return metas, partial, nil +} + +var _ MetaFetcherFilter = (&TimePartitionMetaFilter{}).Filter + +// TimePartitionMetaFilter is a MetaFetcher filter that filters out blocks that are outside of specified time range. +type TimePartitionMetaFilter struct { + minTime, maxTime model.TimeOrDurationValue +} + +// NewTimePartitionMetaFilter creates TimePartitionMetaFilter. +func NewTimePartitionMetaFilter(MinTime, MaxTime model.TimeOrDurationValue) *TimePartitionMetaFilter { + return &TimePartitionMetaFilter{minTime: MinTime, maxTime: MaxTime} +} + +// Filter filters out blocks that are outside of specified time range. +func (f *TimePartitionMetaFilter) Filter(metas map[ulid.ULID]*metadata.Meta, synced GaugeLabeled, _ bool) { + for id, m := range metas { + if m.MaxTime >= f.minTime.PrometheusTimestamp() && m.MinTime <= f.maxTime.PrometheusTimestamp() { + continue + } + synced.WithLabelValues(timeExcludedMeta).Inc() + delete(metas, id) + } +} + +var _ MetaFetcherFilter = (&LabelShardedMetaFilter{}).Filter + +// LabelShardedMetaFilter is a MetaFetcher filter that filters out blocks that have no labels after relabelling. +type LabelShardedMetaFilter struct { + relabelConfig []*relabel.Config +} + +// NewLabelShardedMetaFilter creates LabelShardedMetaFilter. +func NewLabelShardedMetaFilter(relabelConfig []*relabel.Config) *LabelShardedMetaFilter { + return &LabelShardedMetaFilter{relabelConfig: relabelConfig} +} + +// Filter filters out blocks that filters blocks that have no labels after relabelling. +func (f *LabelShardedMetaFilter) Filter(metas map[ulid.ULID]*metadata.Meta, synced GaugeLabeled, _ bool) { + for id, m := range metas { + if processedLabels := relabel.Process(labels.FromMap(m.Thanos.Labels), f.relabelConfig...); processedLabels != nil { + continue + } + synced.WithLabelValues(labelExcludedMeta).Inc() + delete(metas, id) + } +} diff --git a/pkg/block/fetcher_test.go b/pkg/block/fetcher_test.go new file mode 100644 index 0000000000..8be50a928f --- /dev/null +++ b/pkg/block/fetcher_test.go @@ -0,0 +1,392 @@ +package block + +import ( + "bytes" + "context" + "encoding/json" + "io/ioutil" + "os" + "path" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/go-kit/kit/log" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + promtest "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/pkg/relabel" + "github.com/prometheus/prometheus/tsdb" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/model" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/objstore/objtesting" + "github.com/thanos-io/thanos/pkg/testutil" + "gopkg.in/yaml.v2" +) + +func ULID(i int) ulid.ULID { return ulid.MustNew(uint64(i), nil) } + +func ULIDs(is ...int) []ulid.ULID { + ret := []ulid.ULID{} + for _, i := range is { + ret = append(ret, ulid.MustNew(uint64(i), nil)) + } + + return ret +} + +func TestMetaFetcher_Fetch(t *testing.T) { + objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + dir, err := ioutil.TempDir("", "test-meta-fetcher") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() + + var ulidToDelete ulid.ULID + r := prometheus.NewRegistry() + f, err := NewMetaFetcher(log.NewNopLogger(), 20, bkt, dir, r, func(metas map[ulid.ULID]*metadata.Meta, synced GaugeLabeled, incompleteView bool) { + if _, ok := metas[ulidToDelete]; ok { + synced.WithLabelValues("filtered").Inc() + delete(metas, ulidToDelete) + } + }) + testutil.Ok(t, err) + + for i, tcase := range []struct { + name string + do func() + filterULID ulid.ULID + expectedMetas []ulid.ULID + expectedCorruptedMeta []ulid.ULID + expectedNoMeta []ulid.ULID + expectedFiltered int + expectedMetaErr error + }{ + { + name: "empty bucket", + do: func() {}, + + expectedMetas: ULIDs(), + expectedCorruptedMeta: ULIDs(), + expectedNoMeta: ULIDs(), + }, + { + name: "3 metas in bucket", + do: func() { + var meta metadata.Meta + meta.Version = 1 + meta.ULID = ULID(1) + + var buf bytes.Buffer + testutil.Ok(t, json.NewEncoder(&buf).Encode(&meta)) + testutil.Ok(t, bkt.Upload(ctx, path.Join(meta.ULID.String(), metadata.MetaFilename), &buf)) + + meta.ULID = ULID(2) + testutil.Ok(t, json.NewEncoder(&buf).Encode(&meta)) + testutil.Ok(t, bkt.Upload(ctx, path.Join(meta.ULID.String(), metadata.MetaFilename), &buf)) + + meta.ULID = ULID(3) + testutil.Ok(t, json.NewEncoder(&buf).Encode(&meta)) + testutil.Ok(t, bkt.Upload(ctx, path.Join(meta.ULID.String(), metadata.MetaFilename), &buf)) + }, + + expectedMetas: ULIDs(1, 2, 3), + expectedCorruptedMeta: ULIDs(), + expectedNoMeta: ULIDs(), + }, + { + name: "nothing changed", + do: func() {}, + + expectedMetas: ULIDs(1, 2, 3), + expectedCorruptedMeta: ULIDs(), + expectedNoMeta: ULIDs(), + }, + { + name: "fresh cache", + do: func() { + f.cached = map[ulid.ULID]*metadata.Meta{} + }, + + expectedMetas: ULIDs(1, 2, 3), + expectedCorruptedMeta: ULIDs(), + expectedNoMeta: ULIDs(), + }, + { + name: "fresh cache: meta 2 and 3 have corrupted data on disk ", + do: func() { + f.cached = map[ulid.ULID]*metadata.Meta{} + + testutil.Ok(t, os.Remove(filepath.Join(dir, "meta-syncer", ULID(2).String(), MetaFilename))) + + f, err := os.OpenFile(filepath.Join(dir, "meta-syncer", ULID(3).String(), MetaFilename), os.O_WRONLY, os.ModePerm) + testutil.Ok(t, err) + + _, err = f.WriteString("{ almost") + testutil.Ok(t, err) + testutil.Ok(t, f.Close()) + }, + + expectedMetas: ULIDs(1, 2, 3), + expectedCorruptedMeta: ULIDs(), + expectedNoMeta: ULIDs(), + }, + { + name: "block without meta", + do: func() { + testutil.Ok(t, bkt.Upload(ctx, path.Join(ULID(4).String(), "some-file"), bytes.NewBuffer([]byte("something")))) + }, + + expectedMetas: ULIDs(1, 2, 3), + expectedCorruptedMeta: ULIDs(), + expectedNoMeta: ULIDs(4), + }, + { + name: "corrupted meta.json", + do: func() { + testutil.Ok(t, bkt.Upload(ctx, path.Join(ULID(5).String(), MetaFilename), bytes.NewBuffer([]byte("{ not a json")))) + }, + + expectedMetas: ULIDs(1, 2, 3), + expectedCorruptedMeta: ULIDs(5), + expectedNoMeta: ULIDs(4), + }, + { + name: "some added some deleted", + do: func() { + testutil.Ok(t, Delete(ctx, log.NewNopLogger(), bkt, ULID(2))) + + var meta metadata.Meta + meta.Version = 1 + meta.ULID = ULID(6) + + var buf bytes.Buffer + testutil.Ok(t, json.NewEncoder(&buf).Encode(&meta)) + testutil.Ok(t, bkt.Upload(ctx, path.Join(meta.ULID.String(), metadata.MetaFilename), &buf)) + }, + + expectedMetas: ULIDs(1, 3, 6), + expectedCorruptedMeta: ULIDs(5), + expectedNoMeta: ULIDs(4), + }, + { + name: "filter not existing ulid", + do: func() {}, + filterULID: ULID(10), + + expectedMetas: ULIDs(1, 3, 6), + expectedCorruptedMeta: ULIDs(5), + expectedNoMeta: ULIDs(4), + }, + { + name: "filter ulid 1", + do: func() {}, + filterULID: ULID(1), + + expectedMetas: ULIDs(3, 6), + expectedCorruptedMeta: ULIDs(5), + expectedNoMeta: ULIDs(4), + expectedFiltered: 1, + }, + { + name: "error: not supported meta version", + do: func() { + var meta metadata.Meta + meta.Version = 20 + meta.ULID = ULID(7) + + var buf bytes.Buffer + testutil.Ok(t, json.NewEncoder(&buf).Encode(&meta)) + testutil.Ok(t, bkt.Upload(ctx, path.Join(meta.ULID.String(), metadata.MetaFilename), &buf)) + }, + + expectedMetas: ULIDs(1, 3, 6), + expectedCorruptedMeta: ULIDs(5), + expectedNoMeta: ULIDs(4), + expectedMetaErr: errors.New("incomplete view: unexpected meta file: 00000000070000000000000000/meta.json version: 20"), + }, + } { + if ok := t.Run(tcase.name, func(t *testing.T) { + tcase.do() + + ulidToDelete = tcase.filterULID + metas, partial, err := f.Fetch(ctx) + if tcase.expectedMetaErr != nil { + testutil.NotOk(t, err) + testutil.Equals(t, tcase.expectedMetaErr.Error(), err.Error()) + } else { + testutil.Ok(t, err) + } + + { + metasSlice := make([]ulid.ULID, 0, len(metas)) + for id, m := range metas { + testutil.Assert(t, m != nil, "meta is nil") + metasSlice = append(metasSlice, id) + } + sort.Slice(metasSlice, func(i, j int) bool { + return metasSlice[i].Compare(metasSlice[j]) < 0 + }) + testutil.Equals(t, tcase.expectedMetas, metasSlice) + } + + { + partialSlice := make([]ulid.ULID, 0, len(partial)) + for id := range partial { + + partialSlice = append(partialSlice, id) + } + sort.Slice(partialSlice, func(i, j int) bool { + return partialSlice[i].Compare(partialSlice[j]) >= 0 + }) + expected := append([]ulid.ULID{}, tcase.expectedCorruptedMeta...) + expected = append(expected, tcase.expectedNoMeta...) + sort.Slice(expected, func(i, j int) bool { + return expected[i].Compare(expected[j]) >= 0 + }) + testutil.Equals(t, expected, partialSlice) + } + + expectedFailures := 0 + if tcase.expectedMetaErr != nil { + expectedFailures = 1 + } + testutil.Equals(t, float64(i+1), promtest.ToFloat64(f.metrics.syncs)) + testutil.Equals(t, float64(len(tcase.expectedMetas)), promtest.ToFloat64(f.metrics.synced.WithLabelValues(loadedMeta))) + testutil.Equals(t, float64(len(tcase.expectedNoMeta)), promtest.ToFloat64(f.metrics.synced.WithLabelValues(noMeta))) + testutil.Equals(t, float64(tcase.expectedFiltered), promtest.ToFloat64(f.metrics.synced.WithLabelValues("filtered"))) + testutil.Equals(t, 0.0, promtest.ToFloat64(f.metrics.synced.WithLabelValues(labelExcludedMeta))) + testutil.Equals(t, 0.0, promtest.ToFloat64(f.metrics.synced.WithLabelValues(timeExcludedMeta))) + testutil.Equals(t, float64(expectedFailures), promtest.ToFloat64(f.metrics.synced.WithLabelValues(failedMeta))) + testutil.Equals(t, 0.0, promtest.ToFloat64(f.metrics.synced.WithLabelValues(TooFreshMeta))) + }); !ok { + return + } + } + }) +} + +func TestLabelShardedMetaFilter_Filter(t *testing.T) { + relabelContentYaml := ` + - action: drop + regex: "A" + source_labels: + - cluster + - action: keep + regex: "keepme" + source_labels: + - message + ` + var relabelConfig []*relabel.Config + testutil.Ok(t, yaml.Unmarshal([]byte(relabelContentYaml), &relabelConfig)) + + f := NewLabelShardedMetaFilter(relabelConfig) + + input := map[ulid.ULID]*metadata.Meta{ + ULID(1): { + Thanos: metadata.Thanos{ + Labels: map[string]string{"cluster": "B", "message": "keepme"}, + }, + }, + ULID(2): { + Thanos: metadata.Thanos{ + Labels: map[string]string{"something": "A", "message": "keepme"}, + }, + }, + ULID(3): { + Thanos: metadata.Thanos{ + Labels: map[string]string{"cluster": "A", "message": "keepme"}, + }, + }, + ULID(4): { + Thanos: metadata.Thanos{ + Labels: map[string]string{"cluster": "A", "something": "B", "message": "keepme"}, + }, + }, + ULID(5): { + Thanos: metadata.Thanos{ + Labels: map[string]string{"cluster": "B"}, + }, + }, + ULID(6): { + Thanos: metadata.Thanos{ + Labels: map[string]string{"cluster": "B", "message": "keepme"}, + }, + }, + } + expected := map[ulid.ULID]*metadata.Meta{ + ULID(1): input[ULID(1)], + ULID(2): input[ULID(2)], + ULID(6): input[ULID(6)], + } + + synced := prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"state"}) + f.Filter(input, synced, false) + + testutil.Equals(t, 3.0, promtest.ToFloat64(synced.WithLabelValues(labelExcludedMeta))) + testutil.Equals(t, expected, input) + +} + +func TestTimePartitionMetaFilter_Filter(t *testing.T) { + mint := time.Unix(0, 1*time.Millisecond.Nanoseconds()) + maxt := time.Unix(0, 10*time.Millisecond.Nanoseconds()) + f := NewTimePartitionMetaFilter(model.TimeOrDurationValue{Time: &mint}, model.TimeOrDurationValue{Time: &maxt}) + + input := map[ulid.ULID]*metadata.Meta{ + ULID(1): { + BlockMeta: tsdb.BlockMeta{ + MinTime: 0, + MaxTime: 1, + }, + }, + ULID(2): { + BlockMeta: tsdb.BlockMeta{ + MinTime: 1, + MaxTime: 10, + }, + }, + ULID(3): { + BlockMeta: tsdb.BlockMeta{ + MinTime: 2, + MaxTime: 30, + }, + }, + ULID(4): { + BlockMeta: tsdb.BlockMeta{ + MinTime: 0, + MaxTime: 30, + }, + }, + ULID(5): { + BlockMeta: tsdb.BlockMeta{ + MinTime: -1, + MaxTime: 0, + }, + }, + ULID(6): { + BlockMeta: tsdb.BlockMeta{ + MinTime: 20, + MaxTime: 30, + }, + }, + } + expected := map[ulid.ULID]*metadata.Meta{ + ULID(1): input[ULID(1)], + ULID(2): input[ULID(2)], + ULID(3): input[ULID(3)], + ULID(4): input[ULID(4)], + } + + synced := prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"state"}) + f.Filter(input, synced, false) + + testutil.Equals(t, 2.0, promtest.ToFloat64(synced.WithLabelValues(timeExcludedMeta))) + testutil.Equals(t, expected, input) + +} diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 0484aad4fd..9f35f24bff 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -32,7 +32,7 @@ import ( ) func TestSyncer_SyncMetas_e2e(t *testing.T) { - objtesting.ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { + objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() @@ -80,7 +80,7 @@ func TestSyncer_SyncMetas_e2e(t *testing.T) { } func TestSyncer_GarbageCollect_e2e(t *testing.T) { - objtesting.ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { + objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() @@ -196,7 +196,7 @@ func MetricCount(c prometheus.Collector) int { } func TestGroup_Compact_e2e(t *testing.T) { - objtesting.ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { + objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() @@ -511,7 +511,7 @@ func TestSyncer_SyncMetasFilter_e2e(t *testing.T) { extLsets := []labels.Labels{{{Name: "cluster", Value: "A"}}, {{Name: "cluster", Value: "B"}}} - objtesting.ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { + objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() diff --git a/pkg/extprom/tx_gauge.go b/pkg/extprom/tx_gauge.go new file mode 100644 index 0000000000..d0ee7342e6 --- /dev/null +++ b/pkg/extprom/tx_gauge.go @@ -0,0 +1,89 @@ +package extprom + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" +) + +type TxGaugeVec struct { + current *prometheus.GaugeVec + mtx sync.Mutex + newMetricVal func() *prometheus.GaugeVec + + tx *prometheus.GaugeVec +} + +// NewTxGaugeVec is a prometheus.GaugeVec that allows to start atomic metric value transaction. +// It might be useful if long process that wants to update a GaugeVec but wants to build/accumulate those metrics +// in a concurrent way without exposing partial state to Prometheus. +// Caller can also use this as normal GaugeVec. +// +// Additionally it allows to init LabelValues on each transaction. +// NOTE: This is quite naive implementation creating new prometheus.GaugeVec on each `ResetTx`, use wisely. +func NewTxGaugeVec(opts prometheus.GaugeOpts, labelNames []string, initLabelValues ...[]string) *TxGaugeVec { + f := func() *prometheus.GaugeVec { + g := prometheus.NewGaugeVec(opts, labelNames) + for _, vals := range initLabelValues { + g.WithLabelValues(vals...) + } + return g + } + return &TxGaugeVec{ + current: f(), + newMetricVal: f, + } +} + +// ResetTx starts new transaction. Not goroutine-safe. +func (tx *TxGaugeVec) ResetTx() { + tx.tx = tx.newMetricVal() +} + +// Submit atomically and fully applies new values from existing transaction GaugeVec. Not goroutine-safe. +func (tx *TxGaugeVec) Submit() { + if tx.tx == nil { + return + } + + tx.mtx.Lock() + tx.current = tx.tx + tx.mtx.Unlock() +} + +// Describe is used in Register. +func (tx *TxGaugeVec) Describe(ch chan<- *prometheus.Desc) { + tx.mtx.Lock() + defer tx.mtx.Unlock() + + tx.current.Describe(ch) +} + +// Collect is used by Registered. +func (tx *TxGaugeVec) Collect(ch chan<- prometheus.Metric) { + tx.mtx.Lock() + defer tx.mtx.Unlock() + + tx.current.Collect(ch) +} + +// With works as GetMetricWith, but panics where GetMetricWithLabels would have +// returned an error. Not returning an error allows shortcuts like +// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42) +func (tx *TxGaugeVec) With(labels prometheus.Labels) prometheus.Gauge { + if tx.tx == nil { + tx.ResetTx() + } + return tx.tx.With(labels) +} + +// WithLabelValues works as GetMetricWithLabelValues, but panics where +// GetMetricWithLabelValues would have returned an error. Not returning an +// error allows shortcuts like +// myVec.WithLabelValues("404", "GET").Add(42) +func (tx *TxGaugeVec) WithLabelValues(lvs ...string) prometheus.Gauge { + if tx.tx == nil { + tx.ResetTx() + } + return tx.tx.WithLabelValues(lvs...) +} diff --git a/pkg/extprom/tx_gauge_test.go b/pkg/extprom/tx_gauge_test.go new file mode 100644 index 0000000000..a2c3a5988c --- /dev/null +++ b/pkg/extprom/tx_gauge_test.go @@ -0,0 +1,179 @@ +package extprom + +import ( + "fmt" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestTxGaugeVec(t *testing.T) { + g := NewTxGaugeVec(prometheus.GaugeOpts{ + Name: "metric", + }, []string{"a", "b"}, []string{"a1", "b1"}, []string{"a2", "b2"}) + + for _, tcase := range []struct { + name string + txUse func() + exp map[string]float64 + }{ + { + name: "nothing", + txUse: func() {}, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 0, + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": 0, + }, + }, + { + name: "change a=a1,b=b1", + txUse: func() { + g.WithLabelValues("a1", "b1").Inc() + g.WithLabelValues("a1", "b1").Add(0.3) + }, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 1.3, + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": 0, + }, + }, + { + name: "change a=a1,b=b1 again, should return same result", + txUse: func() { + g.WithLabelValues("a1", "b1").Inc() + g.WithLabelValues("a1", "b1").Add(-10) + g.WithLabelValues("a1", "b1").Add(10.3) + }, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 1.3000000000000007, // Say hi to float comparisons. + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": 0, + }, + }, + { + name: "change a=a1,b=b1 again, should return same result", + txUse: func() { + g.WithLabelValues("a1", "b1").Inc() + g.WithLabelValues("a1", "b1").Add(-10) + g.WithLabelValues("a1", "b1").Set(1.3) + }, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 1.3, + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": 0, + }, + }, + { + name: "nothing again", + txUse: func() {}, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 0, + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": 0, + }, + }, + { + name: "change a=aX,b=b1", + txUse: func() { + g.WithLabelValues("aX", "b1").Set(500.2) + }, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 0, + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": 0, + "name:\"a\" value:\"aX\" ,name:\"b\" value:\"b1\" ": 500.2, + }, + }, + { + name: "change a=aX,b=b1", + txUse: func() { + g.WithLabelValues("aX", "b1").Set(500.2) + }, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 0, + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": 0, + "name:\"a\" value:\"aX\" ,name:\"b\" value:\"b1\" ": 500.2, + }, + }, + { + name: "nothing again", + txUse: func() {}, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 0, + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": 0, + }, + }, + { + name: "change 3 metrics", + txUse: func() { + g.WithLabelValues("a1", "b1").Inc() + g.WithLabelValues("a2", "b2").Add(-2) + g.WithLabelValues("a3", "b3").Set(1.1) + }, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 1, + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": -2, + "name:\"a\" value:\"a3\" ,name:\"b\" value:\"b3\" ": 1.1, + }, + }, + { + name: "nothing again", + txUse: func() {}, + exp: map[string]float64{ + "name:\"a\" value:\"a1\" ,name:\"b\" value:\"b1\" ": 0, + "name:\"a\" value:\"a2\" ,name:\"b\" value:\"b2\" ": 0, + }, + }, + } { + if ok := t.Run(tcase.name, func(t *testing.T) { + g.ResetTx() + + tcase.txUse() + g.Submit() + + testutil.Equals(t, tcase.exp, toFloat64(t, g)) + + }); !ok { + return + } + } +} + +// toFloat64 is prometheus/client_golang/prometheus/testutil.ToFloat64 version that works with multiple labelnames. +// NOTE: Be careful on float comparison. +func toFloat64(t *testing.T, c prometheus.Collector) map[string]float64 { + var ( + mChan = make(chan prometheus.Metric) + exp = map[string]float64{} + ) + + go func() { + c.Collect(mChan) + close(mChan) + }() + + for m := range mChan { + pb := &dto.Metric{} + testutil.Ok(t, m.Write(pb)) + if pb.Gauge != nil { + exp[lbToString(pb.GetLabel())] = pb.Gauge.GetValue() + continue + } + if pb.Counter != nil { + exp[lbToString(pb.GetLabel())] = pb.Counter.GetValue() + continue + } + if pb.Untyped != nil { + exp[lbToString(pb.GetLabel())] = pb.Untyped.GetValue() + } + panic(fmt.Errorf("collected a non-gauge/counter/untyped metric: %s", pb)) + } + + return exp +} + +func lbToString(pairs []*dto.LabelPair) string { + var ret []string + for _, r := range pairs { + ret = append(ret, r.String()) + } + return strings.Join(ret, ",") +} diff --git a/pkg/objstore/objtesting/acceptance_e2e_test.go b/pkg/objstore/objtesting/acceptance_e2e_test.go index ce918289f7..1658aca397 100644 --- a/pkg/objstore/objtesting/acceptance_e2e_test.go +++ b/pkg/objstore/objtesting/acceptance_e2e_test.go @@ -16,7 +16,7 @@ import ( // NOTE: This test assumes strong consistency, but in the same way it does not guarantee that if it passes, the // used object store is strongly consistent. func TestObjStore_AcceptanceTest_e2e(t *testing.T) { - ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { + ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { ctx := context.Background() _, err := bkt.Get(ctx, "") diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go index e96ac0ef5b..07d4b352d2 100644 --- a/pkg/objstore/objtesting/foreach.go +++ b/pkg/objstore/objtesting/foreach.go @@ -40,7 +40,7 @@ func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool { // For each it creates a new bucket with a random name and a cleanup function // that deletes it after test was run. // Use THANOS_TEST_OBJSTORE_SKIP to skip explicitly certain object storages. -func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) { +func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) { t.Parallel() // Mandatory Inmem. Not parallel, to detect problem early. diff --git a/pkg/shipper/shipper_e2e_test.go b/pkg/shipper/shipper_e2e_test.go index 362f27f583..d84e1b6d16 100644 --- a/pkg/shipper/shipper_e2e_test.go +++ b/pkg/shipper/shipper_e2e_test.go @@ -27,7 +27,7 @@ import ( ) func TestShipper_SyncBlocks_e2e(t *testing.T) { - objtesting.ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { + objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { dir, err := ioutil.TempDir("", "shipper-e2e-test") testutil.Ok(t, err) defer func() { diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index d7623e0f84..0ea774e87f 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -391,7 +391,7 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { } func TestBucketStore_e2e(t *testing.T) { - objtesting.ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { + objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -439,7 +439,7 @@ func (g naivePartitioner) Partition(length int, rng func(int) (uint64, uint64)) // This tests if our, sometimes concurrent, fetches for different parts works. // Regression test against: https://github.com/thanos-io/thanos/issues/829. func TestBucketStore_ManyParts_e2e(t *testing.T) { - objtesting.ForeachStore(t, func(t testing.TB, bkt objstore.Bucket) { + objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From f024461d43813ac3eba24e3d69d391792b98188a Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Mon, 6 Jan 2020 13:15:36 +0100 Subject: [PATCH 140/257] pkg/discovery: fix unit tests (#1940) Signed-off-by: Simon Pasquier --- pkg/discovery/cache/cache_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/discovery/cache/cache_test.go b/pkg/discovery/cache/cache_test.go index f1d6c1362d..695eef84e3 100644 --- a/pkg/discovery/cache/cache_test.go +++ b/pkg/discovery/cache/cache_test.go @@ -36,9 +36,7 @@ func TestCacheAddresses(t *testing.T) { } got := c.Addresses() - sort.Slice(got, func(i, j int) bool { - return i < j - }) + sort.Strings(got) if !reflect.DeepEqual(got, expected) { t.Errorf("expected %v, want %v", got, expected) } From 162e960fc73812ed3bf4c503804fc0ddc0878169 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 6 Jan 2020 13:00:54 +0000 Subject: [PATCH 141/257] Use block.MetaFetcher in Store Gateway. (#1936) Fixes: https://github.com/thanos-io/thanos/issues/1874 * Corrupted disk cache for meta.json is handled gracefully. * Synchronize was not taking into account deletion by removing meta.json. * Prepare for future implementation of https://thanos.io/proposals/201901-read-write-operations-bucket.md/ * Better observability for syncronize process. * More logs for store startup process. TODO in separate PR: * More observability for index-cache loading / adding time. Signed-off-by: Bartlomiej Plotka --- CHANGELOG.md | 1 + cmd/thanos/store.go | 22 ++++-- docs/components/store.md | 6 +- pkg/store/bucket.go | 126 ++++++++------------------------- pkg/store/bucket_e2e_test.go | 133 +++++++++++++++++------------------ pkg/store/bucket_test.go | 95 +++++++------------------ 6 files changed, 140 insertions(+), 243 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 642fe2707d..508060b1b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Fixed +- []() Store: Improved synchronization of meta JSON files. Store now properly handles corrupted disk cache. Added meta.json sync metrics. - [#1856](https://github.com/thanos-io/thanos/pull/1856) Receive: close DBReadOnly after flushing to fix a memory leak. - [#1882](https://github.com/thanos-io/thanos/pull/1882) Receive: upload to object storage as 'receive' rather than 'sidecar'. - [#1907](https://github.com/thanos-io/thanos/pull/1907) Store: Fixed the duration unit for the metric `thanos_bucket_store_series_gate_duration_seconds`. diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 1ec81d87b6..6937b9f4df 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -11,8 +11,10 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/relabel" + "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" @@ -26,6 +28,8 @@ import ( yaml "gopkg.in/yaml.v2" ) +const fetcherConcurrency = 32 + // registerStore registers a store command. func registerStore(m map[string]setupFunc, app *kingpin.Application) { cmd := app.Command(component.Store.String(), "store node giving access to blocks in a bucket provider. Now supported GCS, S3, Azure, Swift and Tencent COS.") @@ -47,7 +51,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { Default("2GB").Bytes() maxSampleCount := cmd.Flag("store.grpc.series-sample-limit", - "Maximum amount of samples returned via a single Series call. 0 means no limit. NOTE: for efficiency we take 120 as the number of samples in chunk (it cannot be bigger than that), so the actual number of samples might be lower, even though the maximum could be hit."). + "Maximum amount of samples returned via a single Series call. 0 means no limit. NOTE: For efficiency we take 120 as the number of samples in chunk (it cannot be bigger than that), so the actual number of samples might be lower, even though the maximum could be hit."). Default("0").Uint() maxConcurrent := cmd.Flag("store.grpc.series-max-concurrency", "Maximum number of concurrent Series calls.").Default("20").Int() @@ -57,7 +61,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { syncInterval := cmd.Flag("sync-block-duration", "Repeat interval for syncing the blocks between local and remote view."). Default("3m").Duration() - blockSyncConcurrency := cmd.Flag("block-sync-concurrency", "Number of goroutines to use when syncing blocks from object storage."). + blockSyncConcurrency := cmd.Flag("block-sync-concurrency", "Number of goroutines to use when constructing index-cache.json blocks from object storage."). Default("20").Int() minTime := model.TimeOrDuration(cmd.Flag("min-time", "Start of time range limit to serve. Thanos Store will serve only metrics, which happened later than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y."). @@ -128,7 +132,7 @@ func runStore( indexCacheSizeBytes uint64, chunkPoolSizeBytes uint64, maxSampleCount uint64, - maxConcurrent int, + maxConcurrency int, component component.Component, verbose bool, syncInterval time.Duration, @@ -202,19 +206,27 @@ func runStore( return errors.Wrap(err, "create index cache") } + metaFetcher, err := block.NewMetaFetcher(logger, fetcherConcurrency, bkt, dataDir, extprom.WrapRegistererWithPrefix("thanos_", reg), + block.NewTimePartitionMetaFilter(filterConf.MinTime, filterConf.MaxTime).Filter, + block.NewLabelShardedMetaFilter(relabelConfig).Filter, + ) + if err != nil { + return errors.Wrap(err, "meta fetcher") + } + bs, err := store.NewBucketStore( logger, reg, bkt, + metaFetcher, dataDir, indexCache, chunkPoolSizeBytes, maxSampleCount, - maxConcurrent, + maxConcurrency, verbose, blockSyncConcurrency, filterConf, - relabelConfig, advertiseCompatibilityLabel, ) if err != nil { diff --git a/docs/components/store.md b/docs/components/store.md index c25512993c..f071e9475e 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -85,7 +85,7 @@ Flags: for chunks. --store.grpc.series-sample-limit=0 Maximum amount of samples returned via a single - Series call. 0 means no limit. NOTE: for + Series call. 0 means no limit. NOTE: For efficiency we take 120 as the number of samples in chunk (it cannot be bigger than that), so the actual number of samples might be lower, @@ -105,8 +105,8 @@ Flags: --sync-block-duration=3m Repeat interval for syncing the blocks between local and remote view. --block-sync-concurrency=20 - Number of goroutines to use when syncing blocks - from object storage. + Number of goroutines to use when constructing + index-cache.json blocks from object storage. --min-time=0000-01-01T00:00:00Z Start of time range limit to serve. Thanos Store will serve only metrics, which happened diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 6926da769e..8bc4eadb20 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -22,7 +22,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/fileutil" @@ -190,7 +189,7 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { return &m } -// FilterConfig is a configuration, which Store uses for filtering metrics. +// FilterConfig is a configuration, which Store uses for filtering metrics based on time. type FilterConfig struct { MinTime, MaxTime model.TimeOrDurationValue } @@ -201,6 +200,7 @@ type BucketStore struct { logger log.Logger metrics *bucketStoreMetrics bucket objstore.BucketReader + fetcher block.MetadataFetcher dir string indexCache storecache.IndexCache chunkPool *pool.BytesPool @@ -222,9 +222,7 @@ type BucketStore struct { samplesLimiter *Limiter partitioner partitioner - filterConfig *FilterConfig - relabelConfig []*relabel.Config - + filterConfig *FilterConfig advLabelSets []storepb.LabelSet enableCompatibilityLabel bool } @@ -235,6 +233,7 @@ func NewBucketStore( logger log.Logger, reg prometheus.Registerer, bucket objstore.BucketReader, + fetcher block.MetadataFetcher, dir string, indexCache storecache.IndexCache, maxChunkPoolBytes uint64, @@ -242,8 +241,7 @@ func NewBucketStore( maxConcurrent int, debugLogging bool, blockSyncConcurrency int, - filterConf *FilterConfig, - relabelConfig []*relabel.Config, + filterConfig *FilterConfig, enableCompatibilityLabel bool, ) (*BucketStore, error) { if logger == nil { @@ -265,6 +263,7 @@ func NewBucketStore( s := &BucketStore{ logger: logger, bucket: bucket, + fetcher: fetcher, dir: dir, indexCache: indexCache, chunkPool: chunkPool, @@ -272,14 +271,13 @@ func NewBucketStore( blockSets: map[uint64]*bucketBlockSet{}, debugLogging: debugLogging, blockSyncConcurrency: blockSyncConcurrency, + filterConfig: filterConfig, queryGate: gate.NewGate( maxConcurrent, extprom.WrapRegistererWithPrefix("thanos_bucket_store_series_", reg), ), samplesLimiter: NewLimiter(maxSampleCount, metrics.queriesDropped), partitioner: gapBasedPartitioner{maxGapSize: maxGapSize}, - filterConfig: filterConf, - relabelConfig: relabelConfig, enableCompatibilityLabel: enableCompatibilityLabel, } s.metrics = metrics @@ -310,6 +308,12 @@ func (s *BucketStore) Close() (err error) { // SyncBlocks synchronizes the stores state with the Bucket bucket. // It will reuse disk space as persistent cache based on s.dir param. func (s *BucketStore) SyncBlocks(ctx context.Context) error { + metas, _, metaFetchErr := s.fetcher.Fetch(ctx) + // For partial view allow adding new blocks at least. + if metaFetchErr != nil && metas == nil { + return metaFetchErr + } + var wg sync.WaitGroup blockc := make(chan *metadata.Meta) @@ -318,7 +322,6 @@ func (s *BucketStore) SyncBlocks(ctx context.Context) error { go func() { for meta := range blockc { if err := s.addBlock(ctx, meta); err != nil { - level.Warn(s.logger).Log("msg", "loading block failed", "id", meta.ULID, "err", err) continue } } @@ -326,65 +329,33 @@ func (s *BucketStore) SyncBlocks(ctx context.Context) error { }() } - allIDs := map[ulid.ULID]struct{}{} - - err := s.bucket.Iter(ctx, "", func(name string) error { - // Strip trailing slash indicating a directory. - id, err := ulid.Parse(name[:len(name)-1]) - if err != nil { - return nil - } - - bdir := path.Join(s.dir, id.String()) - meta, err := loadMeta(ctx, s.logger, s.bucket, bdir, id) - if err != nil { - return errors.Wrap(err, "load meta") - } - - inRange, err := s.isBlockInMinMaxRange(ctx, meta) - if err != nil { - level.Warn(s.logger).Log("msg", "error parsing block range", "block", id, "err", err) - return os.RemoveAll(bdir) - } - - if !inRange { - return os.RemoveAll(bdir) - } - - // Check for block labels by relabeling. - // If output is empty, the block will be dropped. - if processedLabels := relabel.Process(labels.FromMap(meta.Thanos.Labels), s.relabelConfig...); processedLabels == nil { - level.Debug(s.logger).Log("msg", "ignoring block (drop in relabeling)", "block", id) - return os.RemoveAll(bdir) - } - - allIDs[id] = struct{}{} - + for id, meta := range metas { if b := s.getBlock(id); b != nil { - return nil + continue } select { case <-ctx.Done(): case blockc <- meta: } - return nil - }) + } close(blockc) wg.Wait() - if err != nil { - return errors.Wrap(err, "iter") + if metaFetchErr != nil { + return metaFetchErr } + // Drop all blocks that are no longer present in the bucket. for id := range s.blocks { - if _, ok := allIDs[id]; ok { + if _, ok := metas[id]; ok { continue } if err := s.removeBlock(id); err != nil { - level.Warn(s.logger).Log("msg", "drop outdated block", "block", id, "err", err) + level.Warn(s.logger).Log("msg", "drop outdated block failed", "block", id, "err", err) s.metrics.blockDropFailures.Inc() } + level.Debug(s.logger).Log("msg", "dropped outdated block", "block", id) s.metrics.blockDrops.Inc() } @@ -436,25 +407,6 @@ func (s *BucketStore) InitialSync(ctx context.Context) error { return nil } -func (s *BucketStore) numBlocks() int { - s.mtx.RLock() - defer s.mtx.RUnlock() - return len(s.blocks) -} - -func (s *BucketStore) isBlockInMinMaxRange(ctx context.Context, meta *metadata.Meta) (bool, error) { - // We check for blocks in configured minTime, maxTime range. - switch { - case meta.MaxTime <= s.filterConfig.MinTime.PrometheusTimestamp(): - return false, nil - - case meta.MinTime >= s.filterConfig.MaxTime.PrometheusTimestamp(): - return false, nil - } - - return true, nil -} - func (s *BucketStore) getBlock(id ulid.ULID) *bucketBlock { s.mtx.RLock() defer s.mtx.RUnlock() @@ -463,13 +415,22 @@ func (s *BucketStore) getBlock(id ulid.ULID) *bucketBlock { func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err error) { dir := filepath.Join(s.dir, meta.ULID.String()) + start := time.Now() + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return errors.Wrap(err, "create dir") + } + + level.Debug(s.logger).Log("msg", "loading new block", "id", meta.ULID) defer func() { if err != nil { s.metrics.blockLoadFailures.Inc() if err2 := os.RemoveAll(dir); err2 != nil { level.Warn(s.logger).Log("msg", "failed to remove block we cannot load", "err", err2) } + level.Warn(s.logger).Log("msg", "loading block failed", "elapsed", time.Since(start), "id", meta.ULID, "err", err) + } else { + level.Debug(s.logger).Log("msg", "loaded block", "elapsed", time.Since(start), "id", meta.ULID) } }() s.metrics.blockLoads.Inc() @@ -1231,31 +1192,6 @@ func (b *bucketBlock) indexCacheFilename() string { return path.Join(b.meta.ULID.String(), block.IndexCacheFilename) } -func loadMeta(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID) (*metadata.Meta, error) { - // If we haven't seen the block before or it is missing the meta.json, download it. - if _, err := os.Stat(path.Join(dir, block.MetaFilename)); os.IsNotExist(err) { - if err := os.MkdirAll(dir, 0777); err != nil { - return nil, errors.Wrap(err, "create dir") - } - src := path.Join(id.String(), block.MetaFilename) - - if err := objstore.DownloadFile(ctx, logger, bkt, src, dir); err != nil { - if bkt.IsObjNotFoundErr(errors.Cause(err)) { - level.Debug(logger).Log("msg", "meta file wasn't found. Block not ready or being deleted.", "block", id.String()) - } - return nil, errors.Wrap(err, "download meta.json") - } - } else if err != nil { - return nil, err - } - meta, err := metadata.Read(dir) - if err != nil { - return nil, errors.Wrap(err, "read meta.json") - } - - return meta, err -} - func (b *bucketBlock) loadIndexCacheFile(ctx context.Context) (err error) { cachefn := filepath.Join(b.dir, block.IndexCacheFilename) if err = b.loadIndexCacheFileFromFile(ctx, cachefn); err == nil { diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index 0ea774e87f..ace07aa7b8 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "fmt" "io/ioutil" "os" "path/filepath" @@ -25,11 +26,11 @@ import ( ) var ( - minTime = time.Unix(0, 0) - maxTime, _ = time.Parse(time.RFC3339, "9999-12-31T23:59:59Z") - minTimeDuration = model.TimeOrDurationValue{Time: &minTime} - maxTimeDuration = model.TimeOrDurationValue{Time: &maxTime} - filterConf = &FilterConfig{ + minTime = time.Unix(0, 0) + maxTime, _ = time.Parse(time.RFC3339, "9999-12-31T23:59:59Z") + minTimeDuration = model.TimeOrDurationValue{Time: &minTime} + maxTimeDuration = model.TimeOrDurationValue{Time: &maxTime} + allowAllFilterConf = &FilterConfig{ MinTime: minTimeDuration, MaxTime: maxTimeDuration, } @@ -80,7 +81,7 @@ type storeSuite struct { } func prepareTestBlocks(t testing.TB, now time.Time, count int, dir string, bkt objstore.Bucket, - series []labels.Labels, extLset labels.Labels) (blocks int, minTime, maxTime int64) { + series []labels.Labels, extLset labels.Labels) (minTime, maxTime int64) { ctx := context.Background() logger := log.NewNopLogger() @@ -111,7 +112,6 @@ func prepareTestBlocks(t testing.TB, now time.Time, count int, dir string, bkt o testutil.Ok(t, block.Upload(ctx, logger, bkt, dir1)) testutil.Ok(t, block.Upload(ctx, logger, bkt, dir2)) - blocks += 2 testutil.Ok(t, os.RemoveAll(dir1)) testutil.Ok(t, os.RemoveAll(dir2)) @@ -120,7 +120,7 @@ func prepareTestBlocks(t testing.TB, now time.Time, count int, dir string, bkt o return } -func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, manyParts bool, maxSampleCount uint64, relabelConfig []*relabel.Config) *storeSuite { +func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, manyParts bool, maxSampleCount uint64, relabelConfig []*relabel.Config, filterConf *FilterConfig) *storeSuite { series := []labels.Labels{ labels.FromStrings("a", "1", "b", "1"), labels.FromStrings("a", "1", "b", "2"), @@ -133,7 +133,7 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m } extLset := labels.FromStrings("ext1", "value1") - blocks, minTime, maxTime := prepareTestBlocks(t, time.Now(), 3, dir, bkt, + minTime, maxTime := prepareTestBlocks(t, time.Now(), 3, dir, bkt, series, extLset) s := &storeSuite{ @@ -143,7 +143,27 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m maxTime: maxTime, } - store, err := NewBucketStore(s.logger, nil, bkt, dir, s.cache, 0, maxSampleCount, 20, false, 20, filterConf, relabelConfig, true) + metaFetcher, err := block.NewMetaFetcher(s.logger, 20, bkt, dir, nil, + block.NewTimePartitionMetaFilter(filterConf.MinTime, filterConf.MaxTime).Filter, + block.NewLabelShardedMetaFilter(relabelConfig).Filter, + ) + testutil.Ok(t, err) + + store, err := NewBucketStore( + s.logger, + nil, + bkt, + metaFetcher, + dir, + s.cache, + 0, + maxSampleCount, + 20, + false, + 20, + filterConf, + true, + ) testutil.Ok(t, err) s.store = store @@ -155,10 +175,6 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m defer cancel() testutil.Ok(t, store.SyncBlocks(ctx)) - - if store.numBlocks() < blocks { - t.Fatalf("not all blocks loaded got %v, expected %v", store.numBlocks(), blocks) - } return s } @@ -399,7 +415,7 @@ func TestBucketStore_e2e(t *testing.T) { testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - s := prepareStoreWithTestBlocks(t, dir, bkt, false, 0, emptyRelabelConfig) + s := prepareStoreWithTestBlocks(t, dir, bkt, false, 0, emptyRelabelConfig, allowAllFilterConf) t.Log("Test with no index cache") s.cache.SwapWith(noopCache{}) @@ -447,7 +463,7 @@ func TestBucketStore_ManyParts_e2e(t *testing.T) { testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - s := prepareStoreWithTestBlocks(t, dir, bkt, true, 0, emptyRelabelConfig) + s := prepareStoreWithTestBlocks(t, dir, bkt, true, 0, emptyRelabelConfig, allowAllFilterConf) indexCache, err := storecache.NewInMemoryIndexCacheWithConfig(s.logger, nil, storecache.InMemoryIndexCacheConfig{ MaxItemSize: 1e5, @@ -469,69 +485,46 @@ func TestBucketStore_TimePartitioning_e2e(t *testing.T) { testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() - series := []labels.Labels{ - labels.FromStrings("a", "1", "b", "1"), - labels.FromStrings("a", "1", "b", "1"), - labels.FromStrings("a", "1", "b", "1"), - labels.FromStrings("a", "1", "b", "1"), - labels.FromStrings("a", "1", "b", "2"), - labels.FromStrings("a", "1", "b", "2"), - labels.FromStrings("a", "1", "b", "2"), - labels.FromStrings("a", "1", "b", "2"), - } - extLset := labels.FromStrings("ext1", "value1") - - _, minTime, _ := prepareTestBlocks(t, time.Now(), 3, dir, bkt, series, extLset) - hourAfter := time.Now().Add(1 * time.Hour) filterMaxTime := model.TimeOrDurationValue{Time: &hourAfter} - store, err := NewBucketStore(nil, nil, bkt, dir, noopCache{}, 0, 0, 20, false, 20, - &FilterConfig{ - MinTime: minTimeDuration, - MaxTime: filterMaxTime, - }, emptyRelabelConfig, true) - testutil.Ok(t, err) - - err = store.SyncBlocks(ctx) - testutil.Ok(t, err) + s := prepareStoreWithTestBlocks(t, dir, bkt, false, 241, emptyRelabelConfig, &FilterConfig{ + MinTime: minTimeDuration, + MaxTime: filterMaxTime, + }) + testutil.Ok(t, s.store.SyncBlocks(ctx)) - mint, maxt := store.TimeRange() - testutil.Equals(t, minTime, mint) + mint, maxt := s.store.TimeRange() + testutil.Equals(t, s.minTime, mint) testutil.Equals(t, filterMaxTime.PrometheusTimestamp(), maxt) - for i, tcase := range []struct { - req *storepb.SeriesRequest - expectedLabels [][]storepb.Label - expectedChunks int - }{ - { - req: &storepb.SeriesRequest{ - Matchers: []storepb.LabelMatcher{ - {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "1"}, - }, - MinTime: mint, - MaxTime: timestamp.FromTime(time.Now().AddDate(0, 0, 1)), - }, - expectedLabels: [][]storepb.Label{ - {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, - {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext2", Value: "value2"}}, - }, - // prepareTestBlocks makes 3 chunks containing 2 hour data, - // we should only get 1, as we are filtering. - expectedChunks: 1, + req := &storepb.SeriesRequest{ + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "1"}, }, - } { - t.Log("Run", i) + MinTime: mint, + MaxTime: timestamp.FromTime(time.Now().AddDate(0, 0, 1)), + } - srv := newStoreSeriesServer(ctx) + expectedLabels := [][]storepb.Label{ + {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}}, + {{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}}, + {{Name: "a", Value: "1"}, {Name: "c", Value: "1"}, {Name: "ext2", Value: "value2"}}, + {{Name: "a", Value: "1"}, {Name: "c", Value: "2"}, {Name: "ext2", Value: "value2"}}, + } - testutil.Ok(t, store.Series(tcase.req, srv)) - testutil.Equals(t, len(tcase.expectedLabels), len(srv.SeriesSet)) + s.cache.SwapWith(noopCache{}) + srv := newStoreSeriesServer(ctx) - for i, s := range srv.SeriesSet { - testutil.Equals(t, tcase.expectedLabels[i], s.Labels) - testutil.Equals(t, tcase.expectedChunks, len(s.Chunks)) - } + testutil.Ok(t, s.store.Series(req, srv)) + testutil.Equals(t, len(expectedLabels), len(srv.SeriesSet)) + + for i, s := range srv.SeriesSet { + fmt.Println(s.Labels) + testutil.Equals(t, expectedLabels[i], s.Labels) + + // prepareTestBlocks makes 3 chunks containing 2 hour data, + // we should only get 1, as we are filtering by time. + testutil.Equals(t, 1, len(s.Chunks)) } } diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 8586036f6b..913714f655 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "fmt" "io" "io/ioutil" "math" @@ -19,14 +20,11 @@ import ( "github.com/leanovate/gopter/gen" "github.com/leanovate/gopter/prop" "github.com/oklog/ulid" - prommodel "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" - "github.com/prometheus/prometheus/pkg/timestamp" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact/downsample" - "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/store/storepb" @@ -439,6 +437,7 @@ func TestBucketStore_Info(t *testing.T) { nil, nil, nil, + nil, dir, noopCache{}, 2e5, @@ -446,8 +445,7 @@ func TestBucketStore_Info(t *testing.T) { 0, false, 20, - filterConf, - emptyRelabelConfig, + allowAllFilterConf, true, ) testutil.Ok(t, err) @@ -462,67 +460,6 @@ func TestBucketStore_Info(t *testing.T) { testutil.Equals(t, []storepb.Label(nil), resp.Labels) } -func TestBucketStore_isBlockInMinMaxRange(t *testing.T) { - ctx := context.TODO() - dir, err := ioutil.TempDir("", "block-min-max-test") - testutil.Ok(t, err) - - series := []labels.Labels{labels.FromStrings("a", "1", "b", "1")} - extLset := labels.FromStrings("ext1", "value1") - - // Create a block in range [-2w, -1w]. - id1, err := testutil.CreateBlock(ctx, dir, series, 10, - timestamp.FromTime(time.Now().Add(-14*24*time.Hour)), - timestamp.FromTime(time.Now().Add(-7*24*time.Hour)), - extLset, 0) - testutil.Ok(t, err) - - // Create a block in range [-1w, 0w]. - id2, err := testutil.CreateBlock(ctx, dir, series, 10, - timestamp.FromTime(time.Now().Add(-7*24*time.Hour)), - timestamp.FromTime(time.Now().Add(-0*24*time.Hour)), - extLset, 0) - testutil.Ok(t, err) - - // Create a block in range [+1w, +2w]. - id3, err := testutil.CreateBlock(ctx, dir, series, 10, - timestamp.FromTime(time.Now().Add(7*24*time.Hour)), - timestamp.FromTime(time.Now().Add(14*24*time.Hour)), - extLset, 0) - testutil.Ok(t, err) - - meta1, err := metadata.Read(path.Join(dir, id1.String())) - testutil.Ok(t, err) - meta2, err := metadata.Read(path.Join(dir, id2.String())) - testutil.Ok(t, err) - meta3, err := metadata.Read(path.Join(dir, id3.String())) - testutil.Ok(t, err) - - // Run actual test. - hourBeforeDur := prommodel.Duration(-1 * time.Hour) - hourBefore := model.TimeOrDurationValue{Dur: &hourBeforeDur} - - // bucketStore accepts blocks in range [0, now-1h]. - bucketStore, err := NewBucketStore(nil, nil, inmem.NewBucket(), dir, noopCache{}, 0, 0, 20, false, 20, - &FilterConfig{ - MinTime: minTimeDuration, - MaxTime: hourBefore, - }, emptyRelabelConfig, true) - testutil.Ok(t, err) - - inRange, err := bucketStore.isBlockInMinMaxRange(context.TODO(), meta1) - testutil.Ok(t, err) - testutil.Equals(t, true, inRange) - - inRange, err = bucketStore.isBlockInMinMaxRange(context.TODO(), meta2) - testutil.Ok(t, err) - testutil.Equals(t, true, inRange) - - inRange, err = bucketStore.isBlockInMinMaxRange(context.TODO(), meta3) - testutil.Ok(t, err) - testutil.Equals(t, false, inRange) -} - type recorder struct { mtx sync.Mutex objstore.Bucket @@ -731,11 +668,29 @@ func testSharding(t *testing.T, reuseDisk string, bkt objstore.Bucket, all ...ul testutil.Ok(t, yaml.Unmarshal([]byte(sc.relabel), &relabelConf)) rec := &recorder{Bucket: bkt} - bucketStore, err := NewBucketStore(logger, nil, rec, dir, noopCache{}, 0, 0, 99, false, 20, - filterConf, relabelConf, true) + metaFetcher, err := block.NewMetaFetcher(logger, 20, bkt, dir, nil, + block.NewTimePartitionMetaFilter(allowAllFilterConf.MinTime, allowAllFilterConf.MaxTime).Filter, + block.NewLabelShardedMetaFilter(relabelConf).Filter, + ) + testutil.Ok(t, err) + + bucketStore, err := NewBucketStore( + logger, + nil, + rec, + metaFetcher, + dir, + noopCache{}, + 0, + 0, + 99, + false, + 20, + allowAllFilterConf, + true) testutil.Ok(t, err) - testutil.Ok(t, bucketStore.SyncBlocks(context.Background())) + testutil.Ok(t, bucketStore.InitialSync(context.Background())) // Check "stored" blocks. ids := make([]ulid.ULID, 0, len(bucketStore.blocks)) @@ -761,6 +716,7 @@ func testSharding(t *testing.T, reuseDisk string, bkt objstore.Bucket, all ...ul // Sort records. We load blocks concurrently so operations might be not ordered. sort.Strings(rec.touched) + fmt.Println(cached, sc.expectedIDs, all, rec.touched) if reuseDisk != "" { testutil.Equals(t, expectedTouchedBlockOps(all, sc.expectedIDs, cached), rec.touched) cached = sc.expectedIDs @@ -794,7 +750,6 @@ func expectedTouchedBlockOps(all []ulid.ULID, expected []ulid.ULID, cached []uli } } - ops = append(ops, path.Join(id.String(), block.MetaFilename)) if found { ops = append(ops, path.Join(id.String(), block.IndexCacheFilename), From 497de41b09a48ae229b839d6511ae012b7d6f8ed Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 6 Jan 2020 13:01:29 +0000 Subject: [PATCH 142/257] Use block.MetaFetcher in Compactor. (#1937) Fixes: https://github.com/thanos-io/thanos/issues/1335 Fixes: https://github.com/thanos-io/thanos/issues/1919 Fixes: https://github.com/thanos-io/thanos/issues/1300 * Clean up of meta files are now started only if block which is being uploaded is older than 2 days (only a mitigation). * Blocks without meta.json are handled properly for all compactor phases. * Prepare for future implementation of https://thanos.io/proposals/201901-read-write-operations-bucket.md/ * Added metric for partialUploadAttempt deletions and delayed it. * More tests. Signed-off-by: Bartlomiej Plotka --- cmd/thanos/compact.go | 117 ++++---- cmd/thanos/downsample.go | 31 +- cmd/thanos/main_test.go | 10 +- docs/components/compact.md | 2 +- pkg/block/fetcher.go | 1 - pkg/compact/clean.go | 48 ++++ pkg/compact/clean_test.go | 72 +++++ pkg/compact/compact.go | 269 +++--------------- pkg/compact/compact_e2e_test.go | 155 +--------- pkg/compact/compact_test.go | 42 --- .../downsample/streamed_block_writer.go | 2 +- pkg/compact/retention.go | 22 +- pkg/compact/retention_test.go | 7 +- 13 files changed, 274 insertions(+), 504 deletions(-) create mode 100644 pkg/compact/clean.go create mode 100644 pkg/compact/clean_test.go diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 8d9c4e9222..0dc8c4d58b 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -13,6 +13,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" + "github.com/oklog/ulid" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -23,6 +24,7 @@ import ( "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" @@ -31,6 +33,11 @@ import ( "gopkg.in/alecthomas/kingpin.v2" ) +const ( + metricIndexGenerateName = "thanos_compact_generated_index_total" + metricIndexGenerateHelp = "Total number of generated indexes." +) + var ( compactions = compactionSet{ 1 * time.Hour, @@ -85,7 +92,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) { objStoreConfig := regCommonObjStoreFlags(cmd, "", true) - consistencyDelay := modelDuration(cmd.Flag("consistency-delay", fmt.Sprintf("Minimum age of fresh (non-compacted) blocks before they are being processed. Malformed blocks older than the maximum of consistency-delay and %s will be removed.", compact.MinimumAgeForRemoval)). + consistencyDelay := modelDuration(cmd.Flag("consistency-delay", fmt.Sprintf("Minimum age of fresh (non-compacted) blocks before they are being processed. Malformed blocks older than the maximum of consistency-delay and %v will be removed.", compact.PartialUploadThresholdAge)). Default("30m")) retentionRaw := modelDuration(cmd.Flag("retention.resolution-raw", "How long to retain raw samples in bucket. 0d - disables this retention").Default("0d")) @@ -162,21 +169,28 @@ func runCompact( ) error { halted := prometheus.NewGauge(prometheus.GaugeOpts{ Name: "thanos_compactor_halted", - Help: "Set to 1 if the compactor halted due to an unexpected error", + Help: "Set to 1 if the compactor halted due to an unexpected error.", }) + halted.Set(0) retried := prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_compactor_retries_total", - Help: "Total number of retries after retriable compactor error", + Help: "Total number of retries after retriable compactor error.", }) iterations := prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_compactor_iterations_total", - Help: "Total number of iterations that were executed successfully", + Help: "Total number of iterations that were executed successfully.", }) - halted.Set(0) - - reg.MustRegister(halted) - reg.MustRegister(retried) - reg.MustRegister(iterations) + consistencyDelayMetric := prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "thanos_consistency_delay_seconds", + Help: "Configured consistency delay in seconds.", + }, func() float64 { + return consistencyDelay.Seconds() + }) + partialUploadDeleteAttempts := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "thanos_compactor_aborted_partial_uploads_deletion_attempts_total", + Help: "Total number of started deletions of blocks that are assumed aborted and only partially uploaded.", + }) + reg.MustRegister(halted, retried, iterations, consistencyDelayMetric, partialUploadDeleteAttempts) downsampleMetrics := newDownsampleMetrics(reg) @@ -225,8 +239,15 @@ func runCompact( } }() - sy, err := compact.NewSyncer(logger, reg, bkt, consistencyDelay, - blockSyncConcurrency, acceptMalformedIndex, false, relabelConfig) + metaFetcher, err := block.NewMetaFetcher(logger, 32, bkt, "", extprom.WrapRegistererWithPrefix("thanos_", reg), + block.NewLabelShardedMetaFilter(relabelConfig).Filter, + (&consistencyDelayMetaFilter{logger: logger, consistencyDelay: consistencyDelay}).Filter, + ) + if err != nil { + return errors.Wrap(err, "create meta fetcher") + } + + sy, err := compact.NewSyncer(logger, reg, bkt, metaFetcher, blockSyncConcurrency, acceptMalformedIndex, false) if err != nil { return errors.Wrap(err, "create syncer") } @@ -276,26 +297,24 @@ func runCompact( level.Info(logger).Log("msg", "retention policy of 1 hour aggregated samples is enabled", "duration", retentionByResolution[compact.ResolutionLevel1h]) } - f := func() error { + compactMainFn := func() error { if err := compactor.Compact(ctx); err != nil { return errors.Wrap(err, "compaction failed") } - level.Info(logger).Log("msg", "compaction iterations done") - // TODO(bplotka): Remove "disableDownsampling" once https://github.com/thanos-io/thanos/issues/297 is fixed. if !disableDownsampling { // After all compactions are done, work down the downsampling backlog. // We run two passes of this to ensure that the 1h downsampling is generated // for 5m downsamplings created in the first run. level.Info(logger).Log("msg", "start first pass of downsampling") - if err := downsampleBucket(ctx, logger, downsampleMetrics, bkt, downsamplingDir); err != nil { + if err := downsampleBucket(ctx, logger, downsampleMetrics, bkt, metaFetcher, downsamplingDir); err != nil { return errors.Wrap(err, "first pass of downsampling failed") } level.Info(logger).Log("msg", "start second pass of downsampling") - if err := downsampleBucket(ctx, logger, downsampleMetrics, bkt, downsamplingDir); err != nil { + if err := downsampleBucket(ctx, logger, downsampleMetrics, bkt, metaFetcher, downsamplingDir); err != nil { return errors.Wrap(err, "second pass of downsampling failed") } level.Info(logger).Log("msg", "downsampling iterations done") @@ -303,9 +322,11 @@ func runCompact( level.Warn(logger).Log("msg", "downsampling was explicitly disabled") } - if err := compact.ApplyRetentionPolicyByResolution(ctx, logger, bkt, retentionByResolution); err != nil { + if err := compact.ApplyRetentionPolicyByResolution(ctx, logger, bkt, metaFetcher, retentionByResolution); err != nil { return errors.Wrap(err, fmt.Sprintf("retention failed")) } + + compact.BestEffortCleanAbortedPartialUploads(ctx, logger, metaFetcher, bkt, partialUploadDeleteAttempts) return nil } @@ -314,18 +335,18 @@ func runCompact( // Generate index file. if generateMissingIndexCacheFiles { - if err := genMissingIndexCacheFiles(ctx, logger, reg, bkt, indexCacheDir); err != nil { + if err := genMissingIndexCacheFiles(ctx, logger, reg, bkt, metaFetcher, indexCacheDir); err != nil { return err } } if !wait { - return f() + return compactMainFn() } // --wait=true is specified. return runutil.Repeat(5*time.Minute, ctx.Done(), func() error { - err := f() + err := compactMainFn() if err == nil { iterations.Inc() return nil @@ -363,13 +384,27 @@ func runCompact( return nil } -const ( - metricIndexGenerateName = "thanos_compact_generated_index_total" - metricIndexGenerateHelp = "Total number of generated indexes." -) +type consistencyDelayMetaFilter struct { + logger log.Logger + consistencyDelay time.Duration +} + +func (f *consistencyDelayMetaFilter) Filter(metas map[ulid.ULID]*metadata.Meta, synced block.GaugeLabeled, _ bool) { + for id, meta := range metas { + if ulid.Now()-id.Time() < uint64(f.consistencyDelay/time.Millisecond) && + meta.Thanos.Source != metadata.BucketRepairSource && + meta.Thanos.Source != metadata.CompactorSource && + meta.Thanos.Source != metadata.CompactorRepairSource { + + level.Debug(f.logger).Log("msg", "block is too fresh for now", "block", id) + synced.WithLabelValues(block.TooFreshMeta).Inc() + delete(metas, id) + } + } +} // genMissingIndexCacheFiles scans over all blocks, generates missing index cache files and uploads them to object storage. -func genMissingIndexCacheFiles(ctx context.Context, logger log.Logger, reg *prometheus.Registry, bkt objstore.Bucket, dir string) error { +func genMissingIndexCacheFiles(ctx context.Context, logger log.Logger, reg *prometheus.Registry, bkt objstore.Bucket, fetcher block.MetadataFetcher, dir string) error { genIndex := prometheus.NewCounter(prometheus.CounterOpts{ Name: metricIndexGenerateName, Help: metricIndexGenerateHelp, @@ -391,38 +426,18 @@ func genMissingIndexCacheFiles(ctx context.Context, logger log.Logger, reg *prom level.Info(logger).Log("msg", "start index cache processing") - var metas []*metadata.Meta - - if err := bkt.Iter(ctx, "", func(name string) error { - id, ok := block.IsBlockDir(name) - if !ok { - return nil - } - - meta, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - // Probably not finished block, skip it. - if bkt.IsObjNotFoundErr(errors.Cause(err)) { - level.Warn(logger).Log("msg", "meta file wasn't found", "block", id.String()) - return nil - } - return errors.Wrap(err, "download metadata") - } + metas, _, err := fetcher.Fetch(ctx) + if err != nil { + return errors.Wrap(err, "fetch metas") + } + for _, meta := range metas { // New version of compactor pushes index cache along with data block. // Skip uncompacted blocks. if meta.Compaction.Level == 1 { - return nil + continue } - metas = append(metas, &meta) - - return nil - }); err != nil { - return errors.Wrap(err, "retrieve bucket block metas") - } - - for _, meta := range metas { if err := generateIndexCacheFile(ctx, bkt, logger, dir, meta); err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 5c053d7e6e..00de9e57cb 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -21,6 +21,7 @@ import ( "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" @@ -88,6 +89,11 @@ func runDownsample( return err } + metaFetcher, err := block.NewMetaFetcher(logger, 32, bkt, "", extprom.WrapRegistererWithPrefix("thanos_", reg)) + if err != nil { + return errors.Wrap(err, "create meta fetcher") + } + // Ensure we close up everything properly. defer func() { if err != nil { @@ -107,13 +113,13 @@ func runDownsample( level.Info(logger).Log("msg", "start first pass of downsampling") - if err := downsampleBucket(ctx, logger, metrics, bkt, dataDir); err != nil { + if err := downsampleBucket(ctx, logger, metrics, bkt, metaFetcher, dataDir); err != nil { return errors.Wrap(err, "downsampling failed") } level.Info(logger).Log("msg", "start second pass of downsampling") - if err := downsampleBucket(ctx, logger, metrics, bkt, dataDir); err != nil { + if err := downsampleBucket(ctx, logger, metrics, bkt, metaFetcher, dataDir); err != nil { return errors.Wrap(err, "downsampling failed") } @@ -148,6 +154,7 @@ func downsampleBucket( logger log.Logger, metrics *DownsampleMetrics, bkt objstore.Bucket, + fetcher block.MetadataFetcher, dir string, ) error { if err := os.RemoveAll(dir); err != nil { @@ -163,25 +170,9 @@ func downsampleBucket( } }() - var metas []*metadata.Meta - - err := bkt.Iter(ctx, "", func(name string) error { - id, ok := block.IsBlockDir(name) - if !ok { - return nil - } - - m, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - return errors.Wrap(err, "download metadata") - } - - metas = append(metas, &m) - - return nil - }) + metas, _, err := fetcher.Fetch(ctx) if err != nil { - return errors.Wrap(err, "retrieve bucket block metas") + return errors.Wrap(err, "downsampling meta fetch") } // mapping from a hash over all source IDs to blocks. We don't need to downsample a block diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index ea4f7063e1..f6aedb48eb 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -75,7 +75,10 @@ func TestCleanupIndexCacheFolder(t *testing.T) { }) expReg.MustRegister(genIndexExp) - testutil.Ok(t, genMissingIndexCacheFiles(ctx, logger, reg, bkt, dir)) + metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil) + testutil.Ok(t, err) + + testutil.Ok(t, genMissingIndexCacheFiles(ctx, logger, reg, bkt, metaFetcher, dir)) genIndexExp.Inc() testutil.GatherAndCompare(t, expReg, reg, metricIndexGenerateName) @@ -112,7 +115,10 @@ func TestCleanupDownsampleCacheFolder(t *testing.T) { metrics := newDownsampleMetrics(prometheus.NewRegistry()) testutil.Equals(t, 0.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.GroupKey(meta.Thanos)))) - testutil.Ok(t, downsampleBucket(ctx, logger, metrics, bkt, dir)) + metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil) + testutil.Ok(t, err) + + testutil.Ok(t, downsampleBucket(ctx, logger, metrics, bkt, metaFetcher, dir)) testutil.Equals(t, 1.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.GroupKey(meta.Thanos)))) _, err = os.Stat(dir) diff --git a/docs/components/compact.md b/docs/components/compact.md index 070d82b7b0..440ad9f1c9 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -103,7 +103,7 @@ Flags: --consistency-delay=30m Minimum age of fresh (non-compacted) blocks before they are being processed. Malformed blocks older than the maximum of consistency-delay and - 30m0s will be removed. + 48h0m0s will be removed. --retention.resolution-raw=0d How long to retain raw samples in bucket. 0d - disables this retention diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index 7e2d5a5423..3fc658e3ba 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -223,7 +223,6 @@ func (s *MetaFetcher) loadMeta(ctx context.Context, id ulid.ULID) (*metadata.Met level.Warn(s.logger).Log("msg", "best effort save of the meta.json to local dir failed; ignoring", "dir", cachedBlockDir, "err", err) } } - return m, nil } diff --git a/pkg/compact/clean.go b/pkg/compact/clean.go new file mode 100644 index 0000000000..7fa085eebc --- /dev/null +++ b/pkg/compact/clean.go @@ -0,0 +1,48 @@ +package compact + +import ( + "context" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/oklog/ulid" + "github.com/prometheus/client_golang/prometheus" + "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/objstore" +) + +const ( + // PartialUploadThresholdAge is a time after partial block is assumed aborted and ready to be cleaned. + // Keep it long as it is based on block creation time not upload start time. + PartialUploadThresholdAge = 2 * 24 * time.Hour +) + +func BestEffortCleanAbortedPartialUploads(ctx context.Context, logger log.Logger, fetcher block.MetadataFetcher, bkt objstore.Bucket, deleteAttempts prometheus.Counter) { + level.Info(logger).Log("msg", "started cleaning of aborted partial uploads") + _, partial, err := fetcher.Fetch(ctx) + if err != nil { + level.Warn(logger).Log("msg", "failed to fetch metadata for cleaning of aborted partial uploads; skipping", "err", err) + } + + // Delete partial blocks that are older than partialUploadThresholdAge. + // TODO(bwplotka): This is can cause data loss if blocks are: + // * being uploaded longer than partialUploadThresholdAge + // * being uploaded and started after their partialUploadThresholdAge + // can be assumed in this case. Keep partialUploadThresholdAge long for now. + // Mitigate this by adding ModifiedTime to bkt and check that instead of ULID (block creation time). + for id := range partial { + if ulid.Now()-id.Time() <= uint64(PartialUploadThresholdAge/time.Millisecond) { + // Minimum delay has not expired, ignore for now. + continue + } + + deleteAttempts.Inc() + if err := block.Delete(ctx, logger, bkt, id); err != nil { + level.Warn(logger).Log("msg", "failed to delete aborted partial upload; skipping", "block", id, "thresholdAge", PartialUploadThresholdAge, "err", err) + return + } + level.Info(logger).Log("msg", "deleted aborted partial upload", "block", id, "thresholdAge", PartialUploadThresholdAge) + } + level.Info(logger).Log("msg", "cleaning of aborted partial uploads done") +} diff --git a/pkg/compact/clean_test.go b/pkg/compact/clean_test.go new file mode 100644 index 0000000000..5332da7637 --- /dev/null +++ b/pkg/compact/clean_test.go @@ -0,0 +1,72 @@ +package compact + +import ( + "bytes" + "context" + "encoding/json" + "path" + "testing" + "time" + + "github.com/go-kit/kit/log" + "github.com/oklog/ulid" + "github.com/prometheus/client_golang/prometheus" + promtest "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/objstore/inmem" + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestBestEffortCleanAbortedPartialUploads(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bkt := inmem.NewBucket() + logger := log.NewNopLogger() + + metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil) + testutil.Ok(t, err) + + // 1. No meta, old block, should be removed. + shouldDeleteID, err := ulid.New(uint64(time.Now().Add(-PartialUploadThresholdAge-1*time.Hour).Unix()*1000), nil) + testutil.Ok(t, err) + + var fakeChunk bytes.Buffer + fakeChunk.Write([]byte{0, 1, 2, 3}) + testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldDeleteID.String(), "chunks", "000001"), &fakeChunk)) + + // 2. Old block with meta, so should be kept. + shouldIgnoreID1, err := ulid.New(uint64(time.Now().Add(-PartialUploadThresholdAge-2*time.Hour).Unix()*1000), nil) + testutil.Ok(t, err) + var meta metadata.Meta + meta.Version = 1 + meta.ULID = shouldIgnoreID1 + + var buf bytes.Buffer + testutil.Ok(t, json.NewEncoder(&buf).Encode(&meta)) + testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldIgnoreID1.String(), metadata.MetaFilename), &buf)) + testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldIgnoreID1.String(), "chunks", "000001"), &fakeChunk)) + + // 3. No meta, newer block that should be kept. + shouldIgnoreID2, err := ulid.New(uint64(time.Now().Add(-2*time.Hour).Unix()*1000), nil) + testutil.Ok(t, err) + + testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldIgnoreID2.String(), "chunks", "000001"), &fakeChunk)) + + deleteAttempts := prometheus.NewCounter(prometheus.CounterOpts{}) + BestEffortCleanAbortedPartialUploads(ctx, logger, metaFetcher, bkt, deleteAttempts) + testutil.Equals(t, 1.0, promtest.ToFloat64(deleteAttempts)) + + exists, err := bkt.Exists(ctx, path.Join(shouldDeleteID.String(), "chunks", "000001")) + testutil.Ok(t, err) + testutil.Equals(t, false, exists) + + exists, err = bkt.Exists(ctx, path.Join(shouldIgnoreID1.String(), "chunks", "000001")) + testutil.Ok(t, err) + testutil.Equals(t, true, exists) + + exists, err = bkt.Exists(ctx, path.Join(shouldIgnoreID2.String(), "chunks", "000001")) + testutil.Ok(t, err) + testutil.Equals(t, true, exists) +} diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index b853ff5be1..5b40bca70d 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "os" - "path" "path/filepath" "sort" "sync" @@ -17,7 +16,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb" terrors "github.com/prometheus/prometheus/tsdb/errors" "github.com/thanos-io/thanos/pkg/block" @@ -32,33 +30,24 @@ const ( ResolutionLevelRaw = ResolutionLevel(downsample.ResLevel0) ResolutionLevel5m = ResolutionLevel(downsample.ResLevel1) ResolutionLevel1h = ResolutionLevel(downsample.ResLevel2) - - MinimumAgeForRemoval = time.Duration(30 * time.Minute) ) -var blockTooFreshSentinelError = errors.New("Block too fresh") - -// Syncer syncronizes block metas from a bucket into a local directory. +// Syncer synchronizes block metas from a bucket into a local directory. // It sorts them into compaction groups based on equal label sets. type Syncer struct { logger log.Logger reg prometheus.Registerer bkt objstore.Bucket - consistencyDelay time.Duration + fetcher block.MetadataFetcher mtx sync.Mutex blocks map[ulid.ULID]*metadata.Meta - blocksMtx sync.Mutex blockSyncConcurrency int metrics *syncerMetrics acceptMalformedIndex bool enableVerticalCompaction bool - relabelConfig []*relabel.Config } type syncerMetrics struct { - syncMetas prometheus.Counter - syncMetaFailures prometheus.Counter - syncMetaDuration prometheus.Histogram garbageCollectedBlocks prometheus.Counter garbageCollections prometheus.Counter garbageCollectionFailures prometheus.Counter @@ -73,20 +62,6 @@ type syncerMetrics struct { func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { var m syncerMetrics - m.syncMetas = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "thanos_compact_sync_meta_total", - Help: "Total number of sync meta operations.", - }) - m.syncMetaFailures = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "thanos_compact_sync_meta_failures_total", - Help: "Total number of failed sync meta operations.", - }) - m.syncMetaDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "thanos_compact_sync_meta_duration_seconds", - Help: "Time it took to sync meta files.", - Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720}, - }) - m.garbageCollectedBlocks = prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_compact_garbage_collected_blocks_total", Help: "Total number of deleted blocks by compactor.", @@ -128,9 +103,6 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { if reg != nil { reg.MustRegister( - m.syncMetas, - m.syncMetaFailures, - m.syncMetaDuration, m.garbageCollectedBlocks, m.garbageCollections, m.garbageCollectionFailures, @@ -145,22 +117,21 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { return &m } -// NewSyncer returns a new Syncer for the given Bucket and directory. +// NewMetaSyncer returns a new Syncer for the given Bucket and directory. // Blocks must be at least as old as the sync delay for being considered. -func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, consistencyDelay time.Duration, blockSyncConcurrency int, acceptMalformedIndex bool, enableVerticalCompaction bool, relabelConfig []*relabel.Config) (*Syncer, error) { +func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, fetcher block.MetadataFetcher, blockSyncConcurrency int, acceptMalformedIndex bool, enableVerticalCompaction bool) (*Syncer, error) { if logger == nil { logger = log.NewNopLogger() } return &Syncer{ logger: logger, reg: reg, - consistencyDelay: consistencyDelay, - blocks: map[ulid.ULID]*metadata.Meta{}, bkt: bkt, + fetcher: fetcher, + blocks: map[ulid.ULID]*metadata.Meta{}, metrics: newSyncerMetrics(reg), blockSyncConcurrency: blockSyncConcurrency, acceptMalformedIndex: acceptMalformedIndex, - relabelConfig: relabelConfig, // The syncer offers an option to enable vertical compaction, even if it's // not currently used by Thanos, because the compactor is also used by Cortex // which needs vertical compaction. @@ -168,24 +139,6 @@ func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket }, nil } -// SyncMetas synchronizes all meta files from blocks in the bucket into -// the memory. It removes any partial blocks older than the max of -// consistencyDelay and MinimumAgeForRemoval from the bucket. -func (c *Syncer) SyncMetas(ctx context.Context) error { - c.mtx.Lock() - defer c.mtx.Unlock() - - begin := time.Now() - - err := c.syncMetas(ctx) - if err != nil { - c.metrics.syncMetaFailures.Inc() - } - c.metrics.syncMetas.Inc() - c.metrics.syncMetaDuration.Observe(time.Since(begin).Seconds()) - return err -} - // UntilNextDownsampling calculates how long it will take until the next downsampling operation. // Returns an error if there will be no downsampling. func UntilNextDownsampling(m *metadata.Meta) (time.Duration, error) { @@ -202,155 +155,19 @@ func UntilNextDownsampling(m *metadata.Meta) (time.Duration, error) { } } -func (c *Syncer) syncMetas(ctx context.Context) error { - var wg sync.WaitGroup - defer wg.Wait() - - metaIDsChan := make(chan ulid.ULID) - errChan := make(chan error, c.blockSyncConcurrency) - - workCtx, cancel := context.WithCancel(ctx) - defer cancel() - for i := 0; i < c.blockSyncConcurrency; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - for id := range metaIDsChan { - // Check if we already have this block cached locally. - c.blocksMtx.Lock() - _, seen := c.blocks[id] - c.blocksMtx.Unlock() - if seen { - continue - } - - meta, err := c.downloadMeta(workCtx, id) - if err == blockTooFreshSentinelError { - continue - } - - if err != nil { - if removedOrIgnored := c.removeIfMetaMalformed(workCtx, id); removedOrIgnored { - continue - } - errChan <- err - return - } - - // Check for block labels by relabeling. - // If output is empty, the block will be dropped. - lset := labels.FromMap(meta.Thanos.Labels) - processedLabels := relabel.Process(lset, c.relabelConfig...) - if processedLabels == nil { - level.Debug(c.logger).Log("msg", "dropping block(drop in relabeling)", "block", id) - continue - } - - c.blocksMtx.Lock() - c.blocks[id] = meta - c.blocksMtx.Unlock() - } - }() - } - - // Read back all block metas so we can detect deleted blocks. - remote := map[ulid.ULID]struct{}{} - - err := c.bkt.Iter(ctx, "", func(name string) error { - id, ok := block.IsBlockDir(name) - if !ok { - return nil - } - - remote[id] = struct{}{} +func (s *Syncer) SyncMetas(ctx context.Context) error { + s.mtx.Lock() + defer s.mtx.Unlock() - select { - case <-ctx.Done(): - case metaIDsChan <- id: - } - - return nil - }) - close(metaIDsChan) + metas, _, err := s.fetcher.Fetch(ctx) if err != nil { - return retry(errors.Wrap(err, "retrieve bucket block metas")) - } - - wg.Wait() - close(errChan) - - if err := <-errChan; err != nil { return retry(err) } - - // Delete all local block dirs that no longer exist in the bucket. - for id := range c.blocks { - if _, ok := remote[id]; !ok { - delete(c.blocks, id) - } - } + s.blocks = metas return nil } -func (c *Syncer) downloadMeta(ctx context.Context, id ulid.ULID) (*metadata.Meta, error) { - level.Debug(c.logger).Log("msg", "download meta", "block", id) - - meta, err := block.DownloadMeta(ctx, c.logger, c.bkt, id) - if err != nil { - if ulid.Now()-id.Time() < uint64(c.consistencyDelay/time.Millisecond) { - level.Debug(c.logger).Log("msg", "block is too fresh for now", "block", id) - return nil, blockTooFreshSentinelError - } - return nil, errors.Wrapf(err, "downloading meta.json for %s", id) - } - - // ULIDs contain a millisecond timestamp. We do not consider blocks that have been created too recently to - // avoid races when a block is only partially uploaded. This relates to all blocks, excluding: - // - repair created blocks - // - compactor created blocks - // NOTE: It is not safe to miss "old" block (even that it is newly created) in sync step. Compactor needs to aware of ALL old blocks. - // TODO(bplotka): https://github.com/thanos-io/thanos/issues/377. - if ulid.Now()-id.Time() < uint64(c.consistencyDelay/time.Millisecond) && - meta.Thanos.Source != metadata.BucketRepairSource && - meta.Thanos.Source != metadata.CompactorSource && - meta.Thanos.Source != metadata.CompactorRepairSource { - - level.Debug(c.logger).Log("msg", "block is too fresh for now", "block", id) - return nil, blockTooFreshSentinelError - } - - return &meta, nil -} - -// removeIfMalformed removes a block from the bucket if that block does not have a meta file. It ignores blocks that -// are younger than MinimumAgeForRemoval. -func (c *Syncer) removeIfMetaMalformed(ctx context.Context, id ulid.ULID) (removedOrIgnored bool) { - metaExists, err := c.bkt.Exists(ctx, path.Join(id.String(), block.MetaFilename)) - if err != nil { - level.Warn(c.logger).Log("msg", "failed to check meta exists for block", "block", id, "err", err) - return false - } - if metaExists { - // Meta exists, block is not malformed. - return false - } - - if ulid.Now()-id.Time() <= uint64(MinimumAgeForRemoval/time.Millisecond) { - // Minimum delay has not expired, ignore for now. - return true - } - - if err := block.Delete(ctx, c.logger, c.bkt, id); err != nil { - level.Warn(c.logger).Log("msg", "failed to delete malformed block", "block", id, "err", err) - return false - } - level.Info(c.logger).Log("msg", "deleted malformed block", "block", id) - - return true -} - // GroupKey returns a unique identifier for the group the block belongs to. It considers // the downsampling resolution and the block's labels. func GroupKey(meta metadata.Thanos) string { @@ -363,27 +180,27 @@ func groupKey(res int64, lbls labels.Labels) string { // Groups returns the compaction groups for all blocks currently known to the syncer. // It creates all groups from the scratch on every call. -func (c *Syncer) Groups() (res []*Group, err error) { - c.mtx.Lock() - defer c.mtx.Unlock() +func (s *Syncer) Groups() (res []*Group, err error) { + s.mtx.Lock() + defer s.mtx.Unlock() groups := map[string]*Group{} - for _, m := range c.blocks { + for _, m := range s.blocks { g, ok := groups[GroupKey(m.Thanos)] if !ok { g, err = newGroup( - log.With(c.logger, "compactionGroup", GroupKey(m.Thanos)), - c.bkt, + log.With(s.logger, "compactionGroup", GroupKey(m.Thanos)), + s.bkt, labels.FromMap(m.Thanos.Labels), m.Thanos.Downsample.Resolution, - c.acceptMalformedIndex, - c.enableVerticalCompaction, - c.metrics.compactions.WithLabelValues(GroupKey(m.Thanos)), - c.metrics.compactionRunsStarted.WithLabelValues(GroupKey(m.Thanos)), - c.metrics.compactionRunsCompleted.WithLabelValues(GroupKey(m.Thanos)), - c.metrics.compactionFailures.WithLabelValues(GroupKey(m.Thanos)), - c.metrics.verticalCompactions.WithLabelValues(GroupKey(m.Thanos)), - c.metrics.garbageCollectedBlocks, + s.acceptMalformedIndex, + s.enableVerticalCompaction, + s.metrics.compactions.WithLabelValues(GroupKey(m.Thanos)), + s.metrics.compactionRunsStarted.WithLabelValues(GroupKey(m.Thanos)), + s.metrics.compactionRunsCompleted.WithLabelValues(GroupKey(m.Thanos)), + s.metrics.compactionFailures.WithLabelValues(GroupKey(m.Thanos)), + s.metrics.verticalCompactions.WithLabelValues(GroupKey(m.Thanos)), + s.metrics.garbageCollectedBlocks, ) if err != nil { return nil, errors.Wrap(err, "create compaction group") @@ -403,9 +220,9 @@ func (c *Syncer) Groups() (res []*Group, err error) { // GarbageCollect deletes blocks from the bucket if their data is available as part of a // block with a higher compaction level. -func (c *Syncer) GarbageCollect(ctx context.Context) error { - c.mtx.Lock() - defer c.mtx.Unlock() +func (s *Syncer) GarbageCollect(ctx context.Context) error { + s.mtx.Lock() + defer s.mtx.Unlock() begin := time.Now() @@ -413,12 +230,12 @@ func (c *Syncer) GarbageCollect(ctx context.Context) error { for _, res := range []int64{ downsample.ResLevel0, downsample.ResLevel1, downsample.ResLevel2, } { - err := c.garbageCollect(ctx, res) + err := s.garbageCollect(ctx, res) if err != nil { - c.metrics.garbageCollectionFailures.Inc() + s.metrics.garbageCollectionFailures.Inc() } - c.metrics.garbageCollections.Inc() - c.metrics.garbageCollectionDuration.Observe(time.Since(begin).Seconds()) + s.metrics.garbageCollections.Inc() + s.metrics.garbageCollectionDuration.Observe(time.Since(begin).Seconds()) if err != nil { return errors.Wrapf(err, "garbage collect resolution %d", res) @@ -427,13 +244,12 @@ func (c *Syncer) GarbageCollect(ctx context.Context) error { return nil } -func (c *Syncer) GarbageBlocks(resolution int64) (ids []ulid.ULID, err error) { +func (s *Syncer) GarbageBlocks(resolution int64) (ids []ulid.ULID, err error) { // Map each block to its highest priority parent. Initial blocks have themselves // in their source section, i.e. are their own parent. parents := map[ulid.ULID]ulid.ULID{} - for id, meta := range c.blocks { - + for id, meta := range s.blocks { // Skip any block that has a different resolution. if meta.Thanos.Downsample.Resolution != resolution { continue @@ -447,7 +263,7 @@ func (c *Syncer) GarbageBlocks(resolution int64) (ids []ulid.ULID, err error) { parents[sid] = id continue } - pmeta, ok := c.blocks[pid] + pmeta, ok := s.blocks[pid] if !ok { return nil, errors.Errorf("previous parent block %s not found", pid) } @@ -473,7 +289,7 @@ func (c *Syncer) GarbageBlocks(resolution int64) (ids []ulid.ULID, err error) { topParents[pid] = struct{}{} } - for id, meta := range c.blocks { + for id, meta := range s.blocks { // Skip any block that has a different resolution. if meta.Thanos.Downsample.Resolution != resolution { continue @@ -487,8 +303,8 @@ func (c *Syncer) GarbageBlocks(resolution int64) (ids []ulid.ULID, err error) { return ids, nil } -func (c *Syncer) garbageCollect(ctx context.Context, resolution int64) error { - garbageIds, err := c.GarbageBlocks(resolution) +func (s *Syncer) garbageCollect(ctx context.Context, resolution int64) error { + garbageIds, err := s.GarbageBlocks(resolution) if err != nil { return err } @@ -501,9 +317,9 @@ func (c *Syncer) garbageCollect(ctx context.Context, resolution int64) error { // Spawn a new context so we always delete a block in full on shutdown. delCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - level.Info(c.logger).Log("msg", "deleting outdated block", "block", id) + level.Info(s.logger).Log("msg", "deleting outdated block", "block", id) - err := block.Delete(delCtx, c.logger, c.bkt, id) + err := block.Delete(delCtx, s.logger, s.bkt, id) cancel() if err != nil { return retry(errors.Wrapf(err, "delete block %s from bucket", id)) @@ -511,8 +327,8 @@ func (c *Syncer) garbageCollect(ctx context.Context, resolution int64) error { // Immediately update our in-memory state so no further call to SyncMetas is needed // after running garbage collection. - delete(c.blocks, id) - c.metrics.garbageCollectedBlocks.Inc() + delete(s.blocks, id) + s.metrics.garbageCollectedBlocks.Inc() } return nil } @@ -1161,5 +977,6 @@ func (c *BucketCompactor) Compact(ctx context.Context) error { break } } + level.Info(c.logger).Log("msg", "compaction iterations done") return nil } diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 9f35f24bff..9a7411b2e9 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -20,7 +20,6 @@ import ( "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block" @@ -28,57 +27,8 @@ import ( "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/objtesting" "github.com/thanos-io/thanos/pkg/testutil" - "gopkg.in/yaml.v2" ) -func TestSyncer_SyncMetas_e2e(t *testing.T) { - objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) - defer cancel() - - relabelConfig := make([]*relabel.Config, 0) - sy, err := NewSyncer(nil, nil, bkt, 0, 1, false, false, relabelConfig) - testutil.Ok(t, err) - - // Generate 15 blocks. Initially the first 10 are synced into memory and only the last - // 10 are in the bucket. - // After the first synchronization the first 5 should be dropped and the - // last 5 be loaded from the bucket. - var ids []ulid.ULID - var metas []*metadata.Meta - - for i := 0; i < 15; i++ { - id, err := ulid.New(uint64(i), nil) - testutil.Ok(t, err) - - var meta metadata.Meta - meta.Version = 1 - meta.ULID = id - - if i < 10 { - sy.blocks[id] = &meta - } - ids = append(ids, id) - metas = append(metas, &meta) - } - for _, m := range metas[5:] { - var buf bytes.Buffer - testutil.Ok(t, json.NewEncoder(&buf).Encode(&m)) - testutil.Ok(t, bkt.Upload(ctx, path.Join(m.ULID.String(), metadata.MetaFilename), &buf)) - } - - groups, err := sy.Groups() - testutil.Ok(t, err) - testutil.Equals(t, ids[:10], groups[0].IDs()) - - testutil.Ok(t, sy.SyncMetas(ctx)) - - groups, err = sy.Groups() - testutil.Ok(t, err) - testutil.Equals(t, ids[5:], groups[0].IDs()) - }) -} - func TestSyncer_GarbageCollect_e2e(t *testing.T) { objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) @@ -89,8 +39,6 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) { var metas []*metadata.Meta var ids []ulid.ULID - relabelConfig := make([]*relabel.Config, 0) - for i := 0; i < 10; i++ { var m metadata.Meta @@ -125,7 +73,7 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) { m3.Thanos.Downsample.Resolution = 0 var m4 metadata.Meta - m4.Version = 14 + m4.Version = 1 m4.ULID = ulid.MustNew(400, nil) m4.Compaction.Level = 2 m4.Compaction.Sources = ids[9:] // covers the last block but is a different resolution. Must not trigger deletion. @@ -139,11 +87,14 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) { testutil.Ok(t, bkt.Upload(ctx, path.Join(m.ULID.String(), metadata.MetaFilename), &buf)) } - // Do one initial synchronization with the bucket. - sy, err := NewSyncer(nil, nil, bkt, 0, 1, false, false, relabelConfig) + metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil) testutil.Ok(t, err) - testutil.Ok(t, sy.SyncMetas(ctx)) + sy, err := NewSyncer(nil, nil, bkt, metaFetcher, 1, false, false) + testutil.Ok(t, err) + + // Do one initial synchronization with the bucket. + testutil.Ok(t, sy.SyncMetas(ctx)) testutil.Ok(t, sy.GarbageCollect(ctx)) var rem []ulid.ULID @@ -209,7 +160,10 @@ func TestGroup_Compact_e2e(t *testing.T) { reg := prometheus.NewRegistry() - sy, err := NewSyncer(logger, reg, bkt, 0*time.Second, 5, false, false, nil) + metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil) + testutil.Ok(t, err) + + sy, err := NewSyncer(nil, nil, bkt, metaFetcher, 5, false, false) testutil.Ok(t, err) comp, err := tsdb.NewLeveledCompactor(ctx, reg, logger, []int64{1000, 3000}, nil) @@ -220,8 +174,6 @@ func TestGroup_Compact_e2e(t *testing.T) { // Compaction on empty should not fail. testutil.Ok(t, bComp.Compact(ctx)) - testutil.Equals(t, 1.0, promtest.ToFloat64(sy.metrics.syncMetas)) - testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.syncMetaFailures)) testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.garbageCollectedBlocks)) testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.garbageCollectionFailures)) testutil.Equals(t, 0, MetricCount(sy.metrics.compactions)) @@ -310,8 +262,6 @@ func TestGroup_Compact_e2e(t *testing.T) { }) testutil.Ok(t, bComp.Compact(ctx)) - testutil.Equals(t, 3.0, promtest.ToFloat64(sy.metrics.syncMetas)) - testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.syncMetaFailures)) testutil.Equals(t, 5.0, promtest.ToFloat64(sy.metrics.garbageCollectedBlocks)) testutil.Equals(t, 0.0, promtest.ToFloat64(sy.metrics.garbageCollectionFailures)) testutil.Equals(t, 4, MetricCount(sy.metrics.compactions)) @@ -495,86 +445,3 @@ func createEmptyBlock(dir string, mint int64, maxt int64, extLset labels.Labels, return uid, nil } - -func TestSyncer_SyncMetasFilter_e2e(t *testing.T) { - var err error - - relabelContentYaml := ` - - action: drop - regex: "A" - source_labels: - - cluster - ` - var relabelConfig []*relabel.Config - err = yaml.Unmarshal([]byte(relabelContentYaml), &relabelConfig) - testutil.Ok(t, err) - - extLsets := []labels.Labels{{{Name: "cluster", Value: "A"}}, {{Name: "cluster", Value: "B"}}} - - objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) - defer cancel() - - sy, err := NewSyncer(nil, nil, bkt, 0, 1, false, false, relabelConfig) - testutil.Ok(t, err) - - var ids []ulid.ULID - var metas []*metadata.Meta - - for i := 0; i < 16; i++ { - id, err := ulid.New(uint64(i), nil) - testutil.Ok(t, err) - - var meta metadata.Meta - meta.Version = 1 - meta.ULID = id - meta.Thanos = metadata.Thanos{ - Labels: extLsets[i%2].Map(), - } - - ids = append(ids, id) - metas = append(metas, &meta) - } - for _, m := range metas[:10] { - var buf bytes.Buffer - testutil.Ok(t, json.NewEncoder(&buf).Encode(&m)) - testutil.Ok(t, bkt.Upload(ctx, path.Join(m.ULID.String(), metadata.MetaFilename), &buf)) - } - - testutil.Ok(t, sy.SyncMetas(ctx)) - - groups, err := sy.Groups() - testutil.Ok(t, err) - var evenIds []ulid.ULID - for i := 0; i < 10; i++ { - if i%2 != 0 { - evenIds = append(evenIds, ids[i]) - } - } - testutil.Equals(t, evenIds, groups[0].IDs()) - - // Upload last 6 blocks. - for _, m := range metas[10:] { - var buf bytes.Buffer - testutil.Ok(t, json.NewEncoder(&buf).Encode(&m)) - testutil.Ok(t, bkt.Upload(ctx, path.Join(m.ULID.String(), metadata.MetaFilename), &buf)) - } - - // Delete first 4 blocks. - for _, m := range metas[:4] { - testutil.Ok(t, block.Delete(ctx, log.NewNopLogger(), bkt, m.ULID)) - } - - testutil.Ok(t, sy.SyncMetas(ctx)) - - groups, err = sy.Groups() - testutil.Ok(t, err) - evenIds = make([]ulid.ULID, 0) - for i := 4; i < 16; i++ { - if i%2 != 0 { - evenIds = append(evenIds, ids[i]) - } - } - testutil.Equals(t, evenIds, groups[0].IDs()) - }) -} diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 3624d27fda..f7364f1147 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -1,19 +1,12 @@ package compact import ( - "bytes" - "context" - "path" "testing" - "time" "github.com/thanos-io/thanos/pkg/block/metadata" - "github.com/oklog/ulid" "github.com/pkg/errors" - "github.com/prometheus/prometheus/pkg/relabel" terrors "github.com/prometheus/prometheus/tsdb/errors" - "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -73,41 +66,6 @@ func TestRetryError(t *testing.T) { testutil.Assert(t, IsHaltError(err), "not a halt error. Retry should not hide halt error") } -func TestSyncer_SyncMetas_HandlesMalformedBlocks(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - bkt := inmem.NewBucket() - relabelConfig := make([]*relabel.Config, 0) - sy, err := NewSyncer(nil, nil, bkt, 10*time.Second, 1, false, false, relabelConfig) - testutil.Ok(t, err) - - // Generate 1 block which is older than MinimumAgeForRemoval which has chunk data but no meta. Compactor should delete it. - shouldDeleteId, err := ulid.New(uint64(time.Now().Add(-time.Hour).Unix()*1000), nil) - testutil.Ok(t, err) - - var fakeChunk bytes.Buffer - fakeChunk.Write([]byte{0, 1, 2, 3}) - testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldDeleteId.String(), "chunks", "000001"), &fakeChunk)) - - // Generate 1 block which is older than consistencyDelay but younger than MinimumAgeForRemoval, and which has chunk - // data but no meta. Compactor should ignore it. - shouldIgnoreId, err := ulid.New(uint64(time.Now().Unix()*1000), nil) - testutil.Ok(t, err) - - testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldIgnoreId.String(), "chunks", "000001"), &fakeChunk)) - - testutil.Ok(t, sy.SyncMetas(ctx)) - - exists, err := bkt.Exists(ctx, path.Join(shouldDeleteId.String(), "chunks", "000001")) - testutil.Ok(t, err) - testutil.Equals(t, false, exists) - - exists, err = bkt.Exists(ctx, path.Join(shouldIgnoreId.String(), "chunks", "000001")) - testutil.Ok(t, err) - testutil.Equals(t, true, exists) -} - func TestGroupKey(t *testing.T) { for _, tcase := range []struct { input metadata.Thanos diff --git a/pkg/compact/downsample/streamed_block_writer.go b/pkg/compact/downsample/streamed_block_writer.go index 0bc84e11c4..531ee6efa9 100644 --- a/pkg/compact/downsample/streamed_block_writer.go +++ b/pkg/compact/downsample/streamed_block_writer.go @@ -128,7 +128,7 @@ func NewStreamedBlockWriter( }, nil } -// WriteSeries writes chunks data to the chunkWriter, writes lset and chunks Metas to indexWrites and adds label sets to +// WriteSeries writes chunks data to the chunkWriter, writes lset and chunks MetasFetcher to indexWrites and adds label sets to // labelsValues sets and memPostings to be written on the finalize state in the end of downsampling process. func (w *streamedBlockWriter) WriteSeries(lset labels.Labels, chunks []chunks.Meta) error { if w.finalized || w.ignoreFinalize { diff --git a/pkg/compact/retention.go b/pkg/compact/retention.go index 9021677f2b..aba46963ec 100644 --- a/pkg/compact/retention.go +++ b/pkg/compact/retention.go @@ -13,21 +13,17 @@ import ( // ApplyRetentionPolicyByResolution removes blocks depending on the specified retentionByResolution based on blocks MaxTime. // A value of 0 disables the retention for its resolution. -func ApplyRetentionPolicyByResolution(ctx context.Context, logger log.Logger, bkt objstore.Bucket, retentionByResolution map[ResolutionLevel]time.Duration) error { +func ApplyRetentionPolicyByResolution(ctx context.Context, logger log.Logger, bkt objstore.Bucket, fetcher block.MetadataFetcher, retentionByResolution map[ResolutionLevel]time.Duration) error { level.Info(logger).Log("msg", "start optional retention") - if err := bkt.Iter(ctx, "", func(name string) error { - id, ok := block.IsBlockDir(name) - if !ok { - return nil - } - m, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - return errors.Wrap(err, "download metadata") - } + metas, _, err := fetcher.Fetch(ctx) + if err != nil { + return errors.Wrap(err, "fetch metas") + } + for id, m := range metas { retentionDuration := retentionByResolution[ResolutionLevel(m.Thanos.Downsample.Resolution)] if retentionDuration.Seconds() == 0 { - return nil + continue } maxTime := time.Unix(m.MaxTime/1000, 0) @@ -37,10 +33,6 @@ func ApplyRetentionPolicyByResolution(ctx context.Context, logger log.Logger, bk return errors.Wrap(err, "delete block") } } - - return nil - }); err != nil { - return errors.Wrap(err, "retention") } level.Info(logger).Log("msg", "optional retention apply done") diff --git a/pkg/compact/retention_test.go b/pkg/compact/retention_test.go index c35fde3064..b34847faf3 100644 --- a/pkg/compact/retention_test.go +++ b/pkg/compact/retention_test.go @@ -11,6 +11,7 @@ import ( "github.com/go-kit/kit/log" "github.com/oklog/ulid" "github.com/prometheus/prometheus/tsdb" + "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/objstore" @@ -236,7 +237,11 @@ func TestApplyRetentionPolicyByResolution(t *testing.T) { for _, b := range tt.blocks { uploadMockBlock(t, bkt, b.id, b.minTime, b.maxTime, int64(b.resolution)) } - if err := compact.ApplyRetentionPolicyByResolution(ctx, logger, bkt, tt.retentionByResolution); (err != nil) != tt.wantErr { + + metaFetcher, err := block.NewMetaFetcher(logger, 32, bkt, "", nil) + testutil.Ok(t, err) + + if err := compact.ApplyRetentionPolicyByResolution(ctx, logger, bkt, metaFetcher, tt.retentionByResolution); (err != nil) != tt.wantErr { t.Errorf("ApplyRetentionPolicyByResolution() error = %v, wantErr %v", err, tt.wantErr) } From be72a336facaede40f2f52729cd02cff0f767d08 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 6 Jan 2020 18:06:48 +0000 Subject: [PATCH 143/257] Hide usage of index-cache.json under interface. (#1943) Extra: Ensured LabelNames are sorted. Signed-off-by: Bartlomiej Plotka --- cmd/thanos/compact.go | 3 +- pkg/block/index.go | 221 ------------ pkg/block/index_test.go | 49 --- pkg/block/indexheader/header.go | 19 ++ pkg/block/indexheader/header_test.go | 72 ++++ pkg/block/indexheader/json_reader.go | 315 ++++++++++++++++++ pkg/compact/compact.go | 3 +- .../downsample/streamed_block_writer.go | 3 +- pkg/store/bucket.go | 139 ++------ 9 files changed, 449 insertions(+), 375 deletions(-) delete mode 100644 pkg/block/index_test.go create mode 100644 pkg/block/indexheader/header.go create mode 100644 pkg/block/indexheader/header_test.go create mode 100644 pkg/block/indexheader/json_reader.go diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 0dc8c4d58b..4ff22f0020 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -19,6 +19,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/tsdb" "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/indexheader" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/compact/downsample" @@ -489,7 +490,7 @@ func generateIndexCacheFile( return errors.Wrap(err, "download index file") } - if err := block.WriteIndexCache(logger, indexPath, cachePath); err != nil { + if err := indexheader.WriteJSON(logger, indexPath, cachePath); err != nil { return errors.Wrap(err, "write index cache") } diff --git a/pkg/block/index.go b/pkg/block/index.go index 4d45cc5565..a86abb8045 100644 --- a/pkg/block/index.go +++ b/pkg/block/index.go @@ -1,12 +1,9 @@ package block import ( - "encoding/json" "fmt" "hash/crc32" - "io/ioutil" "math/rand" - "os" "path/filepath" "sort" "strings" @@ -14,8 +11,6 @@ import ( "github.com/thanos-io/thanos/pkg/block/metadata" - "github.com/prometheus/prometheus/tsdb/fileutil" - "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/ulid" @@ -28,222 +23,6 @@ import ( "github.com/thanos-io/thanos/pkg/runutil" ) -const ( - // IndexCacheVersion is a enumeration of index cache versions supported by Thanos. - IndexCacheVersion1 = iota + 1 -) - -var ( - IndexCacheUnmarshalError = errors.New("unmarshal index cache") -) - -type postingsRange struct { - Name, Value string - Start, End int64 -} - -type indexCache struct { - Version int - CacheVersion int - Symbols map[uint32]string - LabelValues map[string][]string - Postings []postingsRange -} - -type realByteSlice []byte - -func (b realByteSlice) Len() int { - return len(b) -} - -func (b realByteSlice) Range(start, end int) []byte { - return b[start:end] -} - -func (b realByteSlice) Sub(start, end int) index.ByteSlice { - return b[start:end] -} - -func getSymbolTable(b index.ByteSlice) (map[uint32]string, error) { - version := int(b.Range(4, 5)[0]) - - if version != 1 && version != 2 { - return nil, errors.Errorf("unknown index file version %d", version) - } - - toc, err := index.NewTOCFromByteSlice(b) - if err != nil { - return nil, errors.Wrap(err, "read TOC") - } - - symbolsV2, symbolsV1, err := index.ReadSymbols(b, version, int(toc.Symbols)) - if err != nil { - return nil, errors.Wrap(err, "read symbols") - } - - symbolsTable := make(map[uint32]string, len(symbolsV1)+len(symbolsV2)) - for o, s := range symbolsV1 { - symbolsTable[o] = s - } - for o, s := range symbolsV2 { - symbolsTable[uint32(o)] = s - } - - return symbolsTable, nil -} - -// WriteIndexCache writes a cache file containing the first lookup stages -// for an index file. -func WriteIndexCache(logger log.Logger, indexFn string, fn string) error { - indexFile, err := fileutil.OpenMmapFile(indexFn) - if err != nil { - return errors.Wrapf(err, "open mmap index file %s", indexFn) - } - defer runutil.CloseWithLogOnErr(logger, indexFile, "close index cache mmap file from %s", indexFn) - - b := realByteSlice(indexFile.Bytes()) - indexr, err := index.NewReader(b) - if err != nil { - return errors.Wrap(err, "open index reader") - } - defer runutil.CloseWithLogOnErr(logger, indexr, "load index cache reader") - - // We assume reader verified index already. - symbols, err := getSymbolTable(b) - if err != nil { - return err - } - - f, err := os.Create(fn) - if err != nil { - return errors.Wrap(err, "create index cache file") - } - defer runutil.CloseWithLogOnErr(logger, f, "index cache writer") - - v := indexCache{ - Version: indexr.Version(), - CacheVersion: IndexCacheVersion1, - Symbols: symbols, - LabelValues: map[string][]string{}, - } - - // Extract label value indices. - lnames, err := indexr.LabelIndices() - if err != nil { - return errors.Wrap(err, "read label indices") - } - for _, lns := range lnames { - if len(lns) != 1 { - continue - } - ln := lns[0] - - tpls, err := indexr.LabelValues(ln) - if err != nil { - return errors.Wrap(err, "get label values") - } - vals := make([]string, 0, tpls.Len()) - - for i := 0; i < tpls.Len(); i++ { - v, err := tpls.At(i) - if err != nil { - return errors.Wrap(err, "get label value") - } - if len(v) != 1 { - return errors.Errorf("unexpected tuple length %d", len(v)) - } - vals = append(vals, v[0]) - } - v.LabelValues[ln] = vals - } - - // Extract postings ranges. - pranges, err := indexr.PostingsRanges() - if err != nil { - return errors.Wrap(err, "read postings ranges") - } - for l, rng := range pranges { - v.Postings = append(v.Postings, postingsRange{ - Name: l.Name, - Value: l.Value, - Start: rng.Start, - End: rng.End, - }) - } - - if err := json.NewEncoder(f).Encode(&v); err != nil { - return errors.Wrap(err, "encode file") - } - return nil -} - -// ReadIndexCache reads an index cache file. -func ReadIndexCache(logger log.Logger, fn string) ( - version int, - symbols []string, - lvals map[string][]string, - postings map[labels.Label]index.Range, - err error, -) { - f, err := os.Open(fn) - if err != nil { - return 0, nil, nil, nil, errors.Wrap(err, "open file") - } - defer runutil.CloseWithLogOnErr(logger, f, "index reader") - - var v indexCache - - bytes, err := ioutil.ReadFile(fn) - if err != nil { - return 0, nil, nil, nil, errors.Wrap(err, "read file") - } - - if err = json.Unmarshal(bytes, &v); err != nil { - return 0, nil, nil, nil, errors.Wrap(IndexCacheUnmarshalError, err.Error()) - } - - strs := map[string]string{} - lvals = make(map[string][]string, len(v.LabelValues)) - postings = make(map[labels.Label]index.Range, len(v.Postings)) - - var maxSymbolID uint32 - for o := range v.Symbols { - if o > maxSymbolID { - maxSymbolID = o - } - } - symbols = make([]string, maxSymbolID+1) - - // Most strings we encounter are duplicates. Dedup string objects that we keep - // around after the function returns to reduce total memory usage. - // NOTE(fabxc): it could even make sense to deduplicate globally. - getStr := func(s string) string { - if cs, ok := strs[s]; ok { - return cs - } - strs[s] = s - return s - } - - for o, s := range v.Symbols { - symbols[o] = getStr(s) - } - for ln, vals := range v.LabelValues { - for i := range vals { - vals[i] = getStr(vals[i]) - } - lvals[getStr(ln)] = vals - } - for _, e := range v.Postings { - l := labels.Label{ - Name: getStr(e.Name), - Value: getStr(e.Value), - } - postings[l] = index.Range{Start: e.Start, End: e.End} - } - return v.Version, symbols, lvals, postings, nil -} - // VerifyIndex does a full run over a block index and verifies that it fulfills the order invariants. func VerifyIndex(logger log.Logger, fn string, minTime int64, maxTime int64) error { stats, err := GatherIndexIssueStats(logger, fn, minTime, maxTime) diff --git a/pkg/block/index_test.go b/pkg/block/index_test.go deleted file mode 100644 index b562ffbe69..0000000000 --- a/pkg/block/index_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package block - -import ( - "context" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/go-kit/kit/log" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/thanos-io/thanos/pkg/testutil" -) - -func TestWriteReadIndexCache(t *testing.T) { - ctx := context.Background() - - tmpDir, err := ioutil.TempDir("", "test-compact-prepare") - testutil.Ok(t, err) - defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() - - b, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ - {{Name: "a", Value: "1"}}, - {{Name: "a", Value: "2"}}, - {{Name: "a", Value: "3"}}, - {{Name: "a", Value: "4"}}, - {{Name: "b", Value: "1"}}, - }, 100, 0, 1000, nil, 124) - testutil.Ok(t, err) - - fn := filepath.Join(tmpDir, "index.cache.json") - testutil.Ok(t, WriteIndexCache(log.NewNopLogger(), filepath.Join(tmpDir, b.String(), "index"), fn)) - - version, symbols, lvals, postings, err := ReadIndexCache(log.NewNopLogger(), fn) - testutil.Ok(t, err) - - testutil.Equals(t, 2, version) - testutil.Equals(t, 6, len(symbols)) - testutil.Equals(t, 2, len(lvals)) - - vals, ok := lvals["a"] - testutil.Assert(t, ok, "") - testutil.Equals(t, []string{"1", "2", "3", "4"}, vals) - - vals, ok = lvals["b"] - testutil.Assert(t, ok, "") - testutil.Equals(t, []string{"1"}, vals) - testutil.Equals(t, 6, len(postings)) -} diff --git a/pkg/block/indexheader/header.go b/pkg/block/indexheader/header.go new file mode 100644 index 0000000000..2f3b2fbbcc --- /dev/null +++ b/pkg/block/indexheader/header.go @@ -0,0 +1,19 @@ +package indexheader + +import ( + "github.com/prometheus/prometheus/tsdb/index" +) + +// NotFoundRange is a range returned by PostingsOffset when there is no posting for given name and value pairs. +// Has to be default value of index.Range. +var NotFoundRange = index.Range{} + +// Reader is an interface allowing to read essential, minimal number of index entries from the small portion of index file called header. +type Reader interface { + IndexVersion() int + // TODO(bwplotka): Move to PostingsOffsets(name string, value ...string) []index.Range and benchmark. + PostingsOffset(name string, value string) index.Range + LookupSymbol(o uint32) (string, error) + LabelValues(name string) []string + LabelNames() []string +} diff --git a/pkg/block/indexheader/header_test.go b/pkg/block/indexheader/header_test.go new file mode 100644 index 0000000000..3b360d126b --- /dev/null +++ b/pkg/block/indexheader/header_test.go @@ -0,0 +1,72 @@ +package indexheader + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/go-kit/kit/log" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/tsdb/index" + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestReaders(t *testing.T) { + ctx := context.Background() + + tmpDir, err := ioutil.TempDir("", "test-indexheader") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() + + b, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + {{Name: "a", Value: "1"}}, + {{Name: "a", Value: "2"}}, + {{Name: "a", Value: "3"}}, + {{Name: "a", Value: "4"}}, + {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}}, + }, 100, 0, 1000, nil, 124) + testutil.Ok(t, err) + + t.Run("JSON", func(t *testing.T) { + fn := filepath.Join(tmpDir, b.String(), "index.cache.json") + testutil.Ok(t, WriteJSON(log.NewNopLogger(), filepath.Join(tmpDir, b.String(), "index"), fn)) + + jr, err := NewJSONReader(ctx, log.NewNopLogger(), nil, tmpDir, b) + testutil.Ok(t, err) + + testutil.Equals(t, 6, len(jr.symbols)) + testutil.Equals(t, 2, len(jr.lvals)) + testutil.Equals(t, 6, len(jr.postings)) + + testReader(t, jr) + }) +} + +func testReader(t *testing.T, r Reader) { + testutil.Equals(t, 2, r.IndexVersion()) + exp := []string{"1", "2", "3", "4", "a", "b"} + for i := range exp { + r, err := r.LookupSymbol(uint32(i)) + testutil.Ok(t, err) + testutil.Equals(t, exp[i], r) + } + _, err := r.LookupSymbol(uint32(len(exp))) + testutil.NotOk(t, err) + + testutil.Equals(t, []string{"1", "2", "3", "4"}, r.LabelValues("a")) + testutil.Equals(t, []string{"1"}, r.LabelValues("b")) + testutil.Equals(t, []string{}, r.LabelValues("c")) + + testutil.Equals(t, []string{"a", "b"}, r.LabelNames()) + + ptr := r.PostingsOffset("a", "1") + testutil.Equals(t, index.Range{Start: 200, End: 212}, ptr) + + ptr = r.PostingsOffset("a", "2") + testutil.Equals(t, index.Range{Start: 220, End: 228}, ptr) + + ptr = r.PostingsOffset("b", "2") + testutil.Equals(t, NotFoundRange, ptr) +} diff --git a/pkg/block/indexheader/json_reader.go b/pkg/block/indexheader/json_reader.go new file mode 100644 index 0000000000..dd79858874 --- /dev/null +++ b/pkg/block/indexheader/json_reader.go @@ -0,0 +1,315 @@ +package indexheader + +import ( + "context" + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "sort" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/tsdb/fileutil" + "github.com/prometheus/prometheus/tsdb/index" + "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/runutil" +) + +const ( + // JSONVersion1 is a enumeration of index-cache.json versions supported by Thanos. + JSONVersion1 = iota + 1 +) + +var ( + jsonUnmarshalError = errors.New("unmarshal index cache") +) + +type postingsRange struct { + Name, Value string + Start, End int64 +} + +type indexCache struct { + Version int + CacheVersion int + Symbols map[uint32]string + LabelValues map[string][]string + Postings []postingsRange +} + +type realByteSlice []byte + +func (b realByteSlice) Len() int { + return len(b) +} + +func (b realByteSlice) Range(start, end int) []byte { + return b[start:end] +} + +func (b realByteSlice) Sub(start, end int) index.ByteSlice { + return b[start:end] +} + +func getSymbolTable(b index.ByteSlice) (map[uint32]string, error) { + version := int(b.Range(4, 5)[0]) + + if version != 1 && version != 2 { + return nil, errors.Errorf("unknown index file version %d", version) + } + + toc, err := index.NewTOCFromByteSlice(b) + if err != nil { + return nil, errors.Wrap(err, "read TOC") + } + + symbolsV2, symbolsV1, err := index.ReadSymbols(b, version, int(toc.Symbols)) + if err != nil { + return nil, errors.Wrap(err, "read symbols") + } + + symbolsTable := make(map[uint32]string, len(symbolsV1)+len(symbolsV2)) + for o, s := range symbolsV1 { + symbolsTable[o] = s + } + for o, s := range symbolsV2 { + symbolsTable[uint32(o)] = s + } + + return symbolsTable, nil +} + +// WriteJSON writes a cache file containing the first lookup stages +// for an index file. +func WriteJSON(logger log.Logger, indexFn string, fn string) error { + indexFile, err := fileutil.OpenMmapFile(indexFn) + if err != nil { + return errors.Wrapf(err, "open mmap index file %s", indexFn) + } + defer runutil.CloseWithLogOnErr(logger, indexFile, "close index cache mmap file from %s", indexFn) + + b := realByteSlice(indexFile.Bytes()) + indexr, err := index.NewReader(b) + if err != nil { + return errors.Wrap(err, "open index reader") + } + defer runutil.CloseWithLogOnErr(logger, indexr, "load index cache reader") + + // We assume reader verified index already. + symbols, err := getSymbolTable(b) + if err != nil { + return err + } + + f, err := os.Create(fn) + if err != nil { + return errors.Wrap(err, "create index cache file") + } + defer runutil.CloseWithLogOnErr(logger, f, "index cache writer") + + v := indexCache{ + Version: indexr.Version(), + CacheVersion: JSONVersion1, + Symbols: symbols, + LabelValues: map[string][]string{}, + } + + // Extract label value indices. + lnames, err := indexr.LabelIndices() + if err != nil { + return errors.Wrap(err, "read label indices") + } + for _, lns := range lnames { + if len(lns) != 1 { + continue + } + ln := lns[0] + + tpls, err := indexr.LabelValues(ln) + if err != nil { + return errors.Wrap(err, "get label values") + } + vals := make([]string, 0, tpls.Len()) + + for i := 0; i < tpls.Len(); i++ { + v, err := tpls.At(i) + if err != nil { + return errors.Wrap(err, "get label value") + } + if len(v) != 1 { + return errors.Errorf("unexpected tuple length %d", len(v)) + } + vals = append(vals, v[0]) + } + v.LabelValues[ln] = vals + } + + // Extract postings ranges. + pranges, err := indexr.PostingsRanges() + if err != nil { + return errors.Wrap(err, "read postings ranges") + } + for l, rng := range pranges { + v.Postings = append(v.Postings, postingsRange{ + Name: l.Name, + Value: l.Value, + Start: rng.Start, + End: rng.End, + }) + } + + if err := json.NewEncoder(f).Encode(&v); err != nil { + return errors.Wrap(err, "encode file") + } + return nil +} + +// JSONReader is a reader based on index-cache.json files. +type JSONReader struct { + indexVersion int + symbols []string + lvals map[string][]string + postings map[labels.Label]index.Range +} + +func NewJSONReader(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID) (*JSONReader, error) { + cachefn := filepath.Join(dir, id.String(), block.IndexCacheFilename) + jr, err := newJSONReaderFromFile(logger, cachefn) + if err == nil { + return jr, err + } + + if !os.IsNotExist(errors.Cause(err)) && errors.Cause(err) != jsonUnmarshalError { + return nil, errors.Wrap(err, "read index cache") + } + + // Try to download index cache file from object store. + if err = objstore.DownloadFile(ctx, logger, bkt, filepath.Join(id.String(), block.IndexCacheFilename), cachefn); err == nil { + return newJSONReaderFromFile(logger, cachefn) + } + + if !bkt.IsObjNotFoundErr(errors.Cause(err)) && errors.Cause(err) != jsonUnmarshalError { + return nil, errors.Wrap(err, "download index cache file") + } + + // No cache exists on disk yet, build it from the downloaded index and retry. + fn := filepath.Join(dir, id.String(), block.IndexFilename) + + if err := objstore.DownloadFile(ctx, logger, bkt, filepath.Join(id.String(), block.IndexFilename), fn); err != nil { + return nil, errors.Wrap(err, "download index file") + } + + defer func() { + if rerr := os.Remove(fn); rerr != nil { + level.Error(logger).Log("msg", "failed to remove temp index file", "path", fn, "err", rerr) + } + }() + + if err := WriteJSON(logger, fn, cachefn); err != nil { + return nil, errors.Wrap(err, "write index cache") + } + + return newJSONReaderFromFile(logger, cachefn) +} + +// ReadJSON reads an index cache file. +func newJSONReaderFromFile(logger log.Logger, fn string) (*JSONReader, error) { + f, err := os.Open(fn) + if err != nil { + return nil, errors.Wrap(err, "open file") + } + defer runutil.CloseWithLogOnErr(logger, f, "index reader") + + var v indexCache + + bytes, err := ioutil.ReadFile(fn) + if err != nil { + return nil, errors.Wrap(err, "read file") + } + + if err = json.Unmarshal(bytes, &v); err != nil { + return nil, errors.Wrap(jsonUnmarshalError, err.Error()) + } + + strs := map[string]string{} + var maxSymbolID uint32 + for o := range v.Symbols { + if o > maxSymbolID { + maxSymbolID = o + } + } + + jr := &JSONReader{ + indexVersion: v.Version, + lvals: make(map[string][]string, len(v.LabelValues)), + postings: make(map[labels.Label]index.Range, len(v.Postings)), + symbols: make([]string, maxSymbolID+1), + } + + // Most strings we encounter are duplicates. Dedup string objects that we keep + // around after the function returns to reduce total memory usage. + // NOTE(fabxc): it could even make sense to deduplicate globally. + getStr := func(s string) string { + if cs, ok := strs[s]; ok { + return cs + } + strs[s] = s + return s + } + + for o, s := range v.Symbols { + jr.symbols[o] = getStr(s) + } + for ln, vals := range v.LabelValues { + for i := range vals { + vals[i] = getStr(vals[i]) + } + jr.lvals[getStr(ln)] = vals + } + for _, e := range v.Postings { + l := labels.Label{ + Name: getStr(e.Name), + Value: getStr(e.Value), + } + jr.postings[l] = index.Range{Start: e.Start, End: e.End} + } + return jr, nil +} + +func (r *JSONReader) IndexVersion() int { + return r.indexVersion +} + +func (r *JSONReader) LookupSymbol(o uint32) (string, error) { + idx := int(o) + if idx >= len(r.symbols) { + return "", errors.Errorf("bucketIndexReader: unknown symbol offset %d", o) + } + + return r.symbols[idx], nil +} + +func (r *JSONReader) PostingsOffset(name, value string) index.Range { + return r.postings[labels.Label{Name: name, Value: value}] +} + +// LabelValues returns label values for single name. +func (r *JSONReader) LabelValues(name string) []string { + res := make([]string, 0, len(r.lvals[name])) + return append(res, r.lvals[name]...) +} + +// LabelNames returns a list of label names. +func (r *JSONReader) LabelNames() []string { + res := make([]string, 0, len(r.lvals)) + for ln := range r.lvals { + res = append(res, ln) + } + sort.Strings(res) + return res +} diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 5b40bca70d..b9f5332fd6 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -19,6 +19,7 @@ import ( "github.com/prometheus/prometheus/tsdb" terrors "github.com/prometheus/prometheus/tsdb/errors" "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/indexheader" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/objstore" @@ -793,7 +794,7 @@ func (cg *Group) compact(ctx context.Context, dir string, comp tsdb.Compactor) ( } } - if err := block.WriteIndexCache(cg.logger, index, indexCache); err != nil { + if err := indexheader.WriteJSON(cg.logger, index, indexCache); err != nil { return false, ulid.ULID{}, errors.Wrap(err, "write index cache") } diff --git a/pkg/compact/downsample/streamed_block_writer.go b/pkg/compact/downsample/streamed_block_writer.go index 531ee6efa9..f5b5b84184 100644 --- a/pkg/compact/downsample/streamed_block_writer.go +++ b/pkg/compact/downsample/streamed_block_writer.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/indexheader" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/runutil" ) @@ -194,7 +195,7 @@ func (w *streamedBlockWriter) Close() error { merr.Add(cl.Close()) } - if err := block.WriteIndexCache( + if err := indexheader.WriteJSON( w.logger, filepath.Join(w.blockDir, block.IndexFilename), filepath.Join(w.blockDir, block.IndexCacheFilename), diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 8bc4eadb20..d069d9c2ef 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -27,6 +27,7 @@ import ( "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/indexheader" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/component" @@ -199,7 +200,7 @@ type FilterConfig struct { type BucketStore struct { logger log.Logger metrics *bucketStoreMetrics - bucket objstore.BucketReader + bkt objstore.BucketReader fetcher block.MetadataFetcher dir string indexCache storecache.IndexCache @@ -262,7 +263,7 @@ func NewBucketStore( metrics := newBucketStoreMetrics(reg) s := &BucketStore{ logger: logger, - bucket: bucket, + bkt: bucket, fetcher: fetcher, dir: dir, indexCache: indexCache, @@ -438,14 +439,20 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er lset := labels.FromMap(meta.Thanos.Labels) h := lset.Hash() + jr, err := indexheader.NewJSONReader(ctx, s.logger, s.bkt, s.dir, meta.ULID) + if err != nil { + return errors.Wrap(err, "create index header reader") + } + b, err := newBucketBlock( ctx, log.With(s.logger, "block", meta.ULID), meta, - s.bucket, + s.bkt, dir, s.indexCache, s.chunkPool, + jr, s.partitioner, ) if err != nil { @@ -952,7 +959,8 @@ func (s *BucketStore) LabelNames(ctx context.Context, _ *storepb.LabelNamesReque g.Go(func() error { defer runutil.CloseWithLogOnErr(s.logger, indexr, "label names") - res := indexr.LabelNames() + // Do it via index reader to have pending reader registered correctly. + res := indexr.block.indexHeaderReader.LabelNames() sort.Strings(res) mtx.Lock() @@ -984,12 +992,11 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR for _, b := range s.blocks { indexr := b.indexReader(gctx) - // TODO(fabxc): only aggregate chunk metas first and add a subsequent fetch stage - // where we consolidate requests. g.Go(func() error { defer runutil.CloseWithLogOnErr(s.logger, indexr, "label values") - res := indexr.LabelValues(req.Label) + // Do it via index reader to have pending reader registered correctly. + res := indexr.block.indexHeaderReader.LabelValues(req.Label) mtx.Lock() sets = append(sets, res) @@ -1139,10 +1146,7 @@ type bucketBlock struct { indexCache storecache.IndexCache chunkPool *pool.BytesPool - indexVersion int - symbols []string - lvals map[string][]string - postings map[labels.Label]index.Range + indexHeaderReader indexheader.Reader chunkObjs []string @@ -1159,20 +1163,20 @@ func newBucketBlock( dir string, indexCache storecache.IndexCache, chunkPool *pool.BytesPool, + indexHeadReader indexheader.Reader, p partitioner, ) (b *bucketBlock, err error) { b = &bucketBlock{ - logger: logger, - bucket: bkt, - indexCache: indexCache, - chunkPool: chunkPool, - dir: dir, - partitioner: p, - meta: meta, - } - if err = b.loadIndexCacheFile(ctx); err != nil { - return nil, errors.Wrap(err, "load index cache") + logger: logger, + bucket: bkt, + indexCache: indexCache, + chunkPool: chunkPool, + dir: dir, + partitioner: p, + meta: meta, + indexHeaderReader: indexHeadReader, } + // Get object handles for all chunk files. err = bkt.Iter(ctx, path.Join(meta.ULID.String(), block.ChunksDirname), func(n string) error { b.chunkObjs = append(b.chunkObjs, n) @@ -1188,53 +1192,6 @@ func (b *bucketBlock) indexFilename() string { return path.Join(b.meta.ULID.String(), block.IndexFilename) } -func (b *bucketBlock) indexCacheFilename() string { - return path.Join(b.meta.ULID.String(), block.IndexCacheFilename) -} - -func (b *bucketBlock) loadIndexCacheFile(ctx context.Context) (err error) { - cachefn := filepath.Join(b.dir, block.IndexCacheFilename) - if err = b.loadIndexCacheFileFromFile(ctx, cachefn); err == nil { - return nil - } - if !os.IsNotExist(errors.Cause(err)) && errors.Cause(err) != block.IndexCacheUnmarshalError { - return errors.Wrap(err, "read index cache") - } - - // Try to download index cache file from object store. - if err = objstore.DownloadFile(ctx, b.logger, b.bucket, b.indexCacheFilename(), cachefn); err == nil { - return b.loadIndexCacheFileFromFile(ctx, cachefn) - } - - if !b.bucket.IsObjNotFoundErr(errors.Cause(err)) && errors.Cause(err) != block.IndexCacheUnmarshalError { - return errors.Wrap(err, "download index cache file") - } - - // No cache exists on disk yet, build it from the downloaded index and retry. - fn := filepath.Join(b.dir, block.IndexFilename) - - if err := objstore.DownloadFile(ctx, b.logger, b.bucket, b.indexFilename(), fn); err != nil { - return errors.Wrap(err, "download index file") - } - - defer func() { - if rerr := os.Remove(fn); rerr != nil { - level.Error(b.logger).Log("msg", "failed to remove temp index file", "path", fn, "err", rerr) - } - }() - - if err := block.WriteIndexCache(b.logger, fn, cachefn); err != nil { - return errors.Wrap(err, "write index cache") - } - - return errors.Wrap(b.loadIndexCacheFileFromFile(ctx, cachefn), "read index cache") -} - -func (b *bucketBlock) loadIndexCacheFileFromFile(ctx context.Context, cache string) (err error) { - b.indexVersion, b.symbols, b.lvals, b.postings, err = block.ReadIndexCache(b.logger, cache) - return err -} - func (b *bucketBlock) readIndexRange(ctx context.Context, off, length int64) ([]byte, error) { r, err := b.bucket.GetRange(ctx, b.indexFilename(), off, length) if err != nil { @@ -1287,7 +1244,8 @@ func (b *bucketBlock) Close() error { return nil } -// bucketIndexReader is a custom index reader (not conforming index.Reader interface) that gets postings. +// bucketIndexReader is a custom index reader (not conforming index.Reader interface) that reads index that is stored in +// object storage without having to fully download it. type bucketIndexReader struct { logger log.Logger ctx context.Context @@ -1302,27 +1260,19 @@ type bucketIndexReader struct { func newBucketIndexReader(ctx context.Context, logger log.Logger, block *bucketBlock, cache storecache.IndexCache) *bucketIndexReader { r := &bucketIndexReader{ - logger: logger, - ctx: ctx, - block: block, - dec: &index.Decoder{}, + logger: logger, + ctx: ctx, + block: block, + dec: &index.Decoder{ + LookupSymbol: block.indexHeaderReader.LookupSymbol, + }, stats: &queryStats{}, cache: cache, loadedSeries: map[uint64][]byte{}, } - r.dec.LookupSymbol = r.lookupSymbol return r } -func (r *bucketIndexReader) lookupSymbol(o uint32) (string, error) { - idx := int(o) - if idx >= len(r.block.symbols) { - return "", errors.Errorf("bucketIndexReader: unknown symbol offset %d", o) - } - - return r.block.symbols[idx], nil -} - // ExpandedPostings returns postings in expanded list instead of index.Postings. // This is because we need to have them buffered anyway to perform efficient lookup // on object storage. @@ -1338,7 +1288,7 @@ func (r *bucketIndexReader) ExpandedPostings(ms []*labels.Matcher) ([]uint64, er // NOTE: Derived from tsdb.PostingsForMatchers. for _, m := range ms { // Each group is separate to tell later what postings are intersecting with what. - postingGroups = append(postingGroups, toPostingGroup(r.LabelValues, m)) + postingGroups = append(postingGroups, toPostingGroup(r.block.indexHeaderReader.LabelValues, m)) } if len(postingGroups) == 0 { @@ -1361,7 +1311,7 @@ func (r *bucketIndexReader) ExpandedPostings(ms []*labels.Matcher) ([]uint64, er // As of version two all series entries are 16 byte padded. All references // we get have to account for that to get the correct offset. - if r.block.indexVersion >= 2 { + if r.block.indexHeaderReader.IndexVersion() >= 2 { for i, id := range ps { ps[i] = id * 16 } @@ -1490,8 +1440,8 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { } // Cache miss; save pointer for actual posting in index stored in object store. - ptr, ok := r.block.postings[key] - if !ok { + ptr := r.block.indexHeaderReader.PostingsOffset(key.Name, key.Value) + if ptr == indexheader.NotFoundRange { // This block does not have any posting for given key. g.Fill(j, index.EmptyPostings()) continue @@ -1684,21 +1634,6 @@ func (r *bucketIndexReader) LoadedSeries(ref uint64, lset *labels.Labels, chks * return r.dec.Series(b, lset, chks) } -// LabelValues returns label values for single name. -func (r *bucketIndexReader) LabelValues(name string) []string { - res := make([]string, 0, len(r.block.lvals[name])) - return append(res, r.block.lvals[name]...) -} - -// LabelNames returns a list of label names. -func (r *bucketIndexReader) LabelNames() []string { - res := make([]string, 0, len(r.block.lvals)) - for ln := range r.block.lvals { - res = append(res, ln) - } - return res -} - // Close released the underlying resources of the reader. func (r *bucketIndexReader) Close() error { r.block.pendingReaders.Done() From 7ff3c08aad5b05c53af4d5e999a641d467992171 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 6 Jan 2020 20:28:31 +0000 Subject: [PATCH 144/257] Unify all CI base images to be built with Go 1.13. (#1945) Master container images were built with Go 1.12 ): Signed-off-by: Bartlomiej Plotka --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c96ebc1da7..a2387bcc71 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,7 +59,7 @@ jobs: publish_master: docker: # Build by Thanos make docker-ci - - image: quay.io/thanos/thanos-ci:v0.1.0 + - image: quay.io/thanos/thanos-ci:v0.2.0 working_directory: /go/src/github.com/thanos-io/thanos steps: - checkout @@ -78,8 +78,8 @@ jobs: publish_release: docker: - # Available from https://hub.docker.com/r/circleci/golang/ - - image: circleci/golang:1.13.1 + # Build by Thanos make docker-ci + - image: quay.io/thanos/thanos-ci:v0.2.0 working_directory: /go/src/github.com/thanos-io/thanos environment: GOBIN: "/go/bin" From 3fa54030c73794abdacff4f63d090ed327e3bf37 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 7 Jan 2020 07:14:39 +0000 Subject: [PATCH 145/257] Updated CHANGELOG and dashboards after meta files synchronizations changes. (#1944) Signed-off-by: Bartlomiej Plotka --- CHANGELOG.md | 10 +++++++++- Makefile | 2 +- examples/dashboards/compactor.json | 10 +++++----- mixin/thanos/dashboards/compactor.libsonnet | 8 ++++---- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 508060b1b3..05b3e5f400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,13 +13,21 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Fixed -- []() Store: Improved synchronization of meta JSON files. Store now properly handles corrupted disk cache. Added meta.json sync metrics. +- [#1919](https://github.com/thanos-io/thanos/issues/1919) Compactor: Fixed potential data loss when uploading older blocks, or upload taking long time while compactor is +running. +- [#1937](https://github.com/thanos-io/thanos/pull/1937) Compactor: Improved synchronization of meta JSON files. +Compactor now properly handles partial block uploads for all operation like retention apply, downsampling and compaction. Additionally: + + * Removed `thanos_compact_sync_meta_*` metrics. Use `thanos_blocks_meta_*` metrics instead. + * Added `thanos_consistency_delay_seconds` and `thanos_compactor_aborted_partial_uploads_deletion_attempts_total` metrics. +- [#1936](https://github.com/thanos-io/thanos/pull/1936) Store: Improved synchronization of meta JSON files. Store now properly handles corrupted disk cache. Added meta.json sync metrics. - [#1856](https://github.com/thanos-io/thanos/pull/1856) Receive: close DBReadOnly after flushing to fix a memory leak. - [#1882](https://github.com/thanos-io/thanos/pull/1882) Receive: upload to object storage as 'receive' rather than 'sidecar'. - [#1907](https://github.com/thanos-io/thanos/pull/1907) Store: Fixed the duration unit for the metric `thanos_bucket_store_series_gate_duration_seconds`. - [#1931](https://github.com/thanos-io/thanos/pull/1931) Compact: Fixed the compactor successfully exiting when actually an error occurred while compacting a blocks group. ### Added + - [#1852](https://github.com/thanos-io/thanos/pull/1852) Add support for `AWS_CONTAINER_CREDENTIALS_FULL_URI` by upgrading to minio-go v6.0.44 - [#1854](https://github.com/thanos-io/thanos/pull/1854) Update Rule UI to support alerts count displaying and filtering. - [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add TLS and authentication support for Alertmanager with the `--alertmanagers.config` and `--alertmanagers.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. diff --git a/Makefile b/Makefile index ae67064ca2..a29e8a16e6 100644 --- a/Makefile +++ b/Makefile @@ -298,7 +298,7 @@ lint: check-git $(GOLANGCILINT) $(MISSPELL) @find . -type f | grep -v vendor/ | grep -vE '\./\..*' | xargs $(MISSPELL) -error @echo ">> detecting white noise" @find . -type f \( -name "*.md" -o -name "*.go" \) | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh - $(call require_clean_work_tree,"lint") + $(call require_clean_work_tree,"detected white noise") .PHONY: web-serve web-serve: web-pre-process $(HUGO) diff --git a/examples/dashboards/compactor.json b/examples/dashboards/compactor.json index 9e2d3d65cc..ba36ef58db 100644 --- a/examples/dashboards/compactor.json +++ b/examples/dashboards/compactor.json @@ -643,7 +643,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(thanos_compact_sync_meta_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "expr": "sum(rate(thanos_blocks_meta_syncs_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "sync {{job}}", @@ -722,7 +722,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(thanos_compact_sync_meta_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_compact_sync_meta_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "expr": "sum(rate(thanos_blocks_meta_sync_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_blocks_meta_syncs_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", @@ -799,7 +799,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(thanos_compact_sync_meta_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(thanos_blocks_meta_sync_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "P99 {{job}}", @@ -807,7 +807,7 @@ "step": 10 }, { - "expr": "sum(rate(thanos_compact_sync_meta_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_compact_sync_meta_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "expr": "sum(rate(thanos_blocks_meta_sync_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_blocks_meta_sync_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "mean {{job}}", @@ -815,7 +815,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(thanos_compact_sync_meta_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(thanos_blocks_meta_sync_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "P50 {{job}}", diff --git a/mixin/thanos/dashboards/compactor.libsonnet b/mixin/thanos/dashboards/compactor.libsonnet index ff45c6dbf2..af34618e04 100644 --- a/mixin/thanos/dashboards/compactor.libsonnet +++ b/mixin/thanos/dashboards/compactor.libsonnet @@ -88,7 +88,7 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; 'Shows rate of execution for all meta files from blocks in the bucket into the memory.' ) + g.queryPanel( - 'sum(rate(thanos_compact_sync_meta_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)', + 'sum(rate(thanos_blocks_meta_syncs_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)', 'sync {{job}}' ) + g.stack @@ -96,13 +96,13 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; .addPanel( g.panel('Errors', 'Shows ratio of errors compared to the total number of executed meta file sync.') + g.qpsErrTotalPanel( - 'thanos_compact_sync_meta_failures_total{namespace="$namespace",job=~"$job"}', - 'thanos_compact_sync_meta_total{namespace="$namespace",job=~"$job"}', + 'thanos_blocks_meta_sync_failures_total{namespace="$namespace",job=~"$job"}', + 'thanos_blocks_meta_syncs_total{namespace="$namespace",job=~"$job"}', ) ) .addPanel( g.panel('Duration', 'Shows how long has it taken to execute meta file sync, in quantiles.') + - g.latencyPanel('thanos_compact_sync_meta_duration_seconds', 'namespace="$namespace",job=~"$job"') + g.latencyPanel('thanos_blocks_meta_sync_duration_seconds', 'namespace="$namespace",job=~"$job"') ) ) .addRow( From d9e4e0e6d7e2bb7370a8d05d5b6e58ee609288e0 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 7 Jan 2020 14:56:53 +0000 Subject: [PATCH 146/257] Added proposal for moving to index-header binary format. (#1839) * Added proposal for moving to index-header binary format. Signed-off-by: Bartek Plotka * Addressed comments, added impactful risk. Signed-off-by: Bartek Plotka * More comments. Signed-off-by: Bartlomiej Plotka --- docs/proposals/201909_thanos_sharding.md | 2 +- .../201912_thanos_binary_index_header.md | 217 ++++++++++++++++++ 2 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 docs/proposals/201912_thanos_binary_index_header.md diff --git a/docs/proposals/201909_thanos_sharding.md b/docs/proposals/201909_thanos_sharding.md index 5ff2cd126a..55e8836722 100644 --- a/docs/proposals/201909_thanos_sharding.md +++ b/docs/proposals/201909_thanos_sharding.md @@ -2,7 +2,7 @@ title: Thanos Sharding for Long Term Retention Storage type: proposal menu: proposals -status: approved +status: accepted owner: bwplotka --- diff --git a/docs/proposals/201912_thanos_binary_index_header.md b/docs/proposals/201912_thanos_binary_index_header.md new file mode 100644 index 0000000000..a1cef048b2 --- /dev/null +++ b/docs/proposals/201912_thanos_binary_index_header.md @@ -0,0 +1,217 @@ +--- +title: Binary Format for Index-cache; Renaming to Index-header. +type: proposal +menu: proposals +status: proposed +owner: bwplotka +--- + +### Related Tickets + +* https://github.com/thanos-io/thanos/issues/942 (format changes) +* https://github.com/thanos-io/thanos/issues/1711 (pulling index in smarter way) +* https://github.com/thanos-io/thanos/pull/1013 (initial investigation of different formats) +* https://github.com/thanos-io/thanos/issues/448 (OOM) +* https://github.com/thanos-io/thanos/issues/1705 (LTS umbrella issue) + +## Summary + +This short document describes the motivation and the design of a new format that is meant to replace `index-cache.json` we have currently. + +We also propose renaming index-cache to *index-header* due to name collision with index cache for postings and series. + +## Motivation + +Currently the Store Gateway component has to be aware of all the blocks (modulo sharding & time partitioning) in the bucket. +For each block that we want to serve metrics from, the Store Gateway has to have that block `synced` in order to: + +* Know what postings are matching given query matchers. +* Know what label values are for names. +* Know how to un-intern symbols. + +The `sync` process includes: + +* Download meta.json or fetch from disk if cached. +* Check if index-cache.json is present on the disk. If not: + * Check if index-cache.json is present in the bucket. If not: + * Download the whole TSDB index file, mmap all of it. + * Build index-cache.json + * else: + * Download index-cache.json from bucket + * Delete downloaded TSDB index. +* Load whole index-cache.json to memory and keep it for lifetime of the block. + +The current, mentioned [index-cache.json][index-cache-code] holds block’s: +* [TOC][toc] +* [Symbols][symbols] +* LabelValues: + * Calculated from all [LabelsIndicesTable][labels-indices-table], that each points to [LabelIndex][label-index] entry. + * Used for matching against request selectors and LabelValues/LabelNames calls. +* [Postings Offsets][postings-offsets] + +There are few problems with this approach: + +1. Downloading the whole TSDB index file (which can be from few MB up to 64GB or even more when limit will be increased soon) is expensive and might be slow. +2. Building such index-cache requires at least 2x `index-cache.json` size of memory. +3. Parsing large index-cache.json files (which can be from few KB to few GB) might be slow. +4. We keep the full content of index-cache file in memory for all blocks. + * Some blocks are never queried. + * Some posting offsets and label values within each block are never used. + +1, 2 & 3 contributes to Store Gateway startup being slow and resource consuming: https://github.com/thanos-io/thanos/issues/448 + +4. causes unnecessary constant cache for unused blocks. + +This design is trying to address those four problems. + +## Goals + +* Reduce confusion between index-cache.json and IndexCache for series and postings. +* Use constant amount of memory for building index-header/index-cache; download only required pieces of index from bucket and compute new TOC file. +* Leverage mmap and let OS unload unused pages from the physical memory as needed to satisfy the queries against the block. We can leverage mmap due to random access against label values and postings offsets. + +## No Goals + +* Removing initial startup for Thanos Store Gateway completely as designed in [Cortex, no initial block sync](https://github.com/thanos-io/thanos/issues/1813) + * However this proposal might be a step towards that as we might be able to load and build index-cache/index quickly on demand from disk. See [Future Work](./201912_thanos_binary_index_header.md#Future-work) + * At the same time be able to load `index-header` at query time directly from bucket is not a goal of this proposal. +* Decreasing size. While it would be nice to use less space; our aim is latency of building/loading the block. That might be correlated with size, but not necessarily (e.g when extra considering compression) + +## Verification + +* Compare memory and latency usage for startup without index-header being generated on compactor +* Benchmark for loading the `index-header` vs `index-cache.json` into Store GW. + +## Proposal + +TSDB index is in binary [format](https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md). + +To allow reduced resource consumption and effort when building (1), (2), "index-header" for blocks we plan to reuse similar format for sections like symbols, label indices and +posting offsets in separate the file called `index-header` that will replace currently existing `index-cache.json`. + +The process for building this will be as follows: + +* Thanks to https://github.com/thanos-io/thanos/pull/1792 we can check final size of index and scan for TOC file. +* With TOC: + * Get the symbols table and copy it to the local file. + * Get the posting table and copy it to the local file. +* Write new TOC file at the end of file. + +With that effort building time and resources should be compared with downloading the prebuilt `index-header` from the bucket. +This allows us to reduce complexity of the system as we don't need to cache that in object storage by compactor. + +Thanks to this format we can reuse most of the [FileReader](https://github.com/prometheus/prometheus/blob/de0a772b8e7d27dc744810a1a693d97be027049a/tsdb/index/index.go#L664) code +to load file. + +Thanos will build/compose all index-headers on the startup for now, however in theory we can load and build those blocks on demand. +Given the minimal memory that each loaded block should take now, this is described as [Future Work](./201912_thanos_binary_index_header.md#Future-Work) + +### Length of Posting to fetch + +While idea of combing different pieces of TSDB index as our index-header is great, unfortunately we heavily rely +on information about size of each posting represented as `postingRange.End`. + +We need to know that apriori to know how to partition and how many bytes we need to fetch from the storage to get each posting: https://github.com/thanos-io/thanos/blob/7e11afe64af0c096743a3de8a594616abf52be45/pkg/store/bucket.go#L1567 + +To calculate those sizes we use [`indexr.PostingsRanges()`](https://github.com/thanos-io/thanos/blob/7e11afe64af0c096743a3de8a594616abf52be45/pkg/block/index.go#L156-L155) which +scans through `posting` section of the TSDB index. Having to fetch whole postings section just to get size of each posting +makes this proposal less valuable as we still need to download big part of index and traverse through it instead of what we propose in [#Proposal](./201912_thanos_binary_index_header.md#proposal) + +For series we don't know the exact size either, however we estimate max size of each series to be 64*1024. It depends on sane number of label pairs and chunks per series. +We have really only one potential case when this was too low: https://github.com/thanos-io/thanos/issues/552. Decision about series this was made here: https://github.com/thanos-io/thanos/issues/146 + +For postings it's more tricky as it depends on number of series in which given label pair exists. For worst case it can be even million label-pair for popular pairs like `__name__=http_request_http_duration_bucket` etc. + +We have few options: + +* Encode this posting size in TSDB index `PostingOffset`: Unlikely to happen as not needed by Prometheus. +* Scan postings to fetch, which is something we wanted to avoid when building `index-header` without downloading full TSDB index: This option invalidates this proposal. +* Estimate some large value (will overfetch): Too large to always overfetch. +* Estimate casual value and retry fetching remaining size on demand for data consistency. + +However there is one that this proposal aims for: + +* We can deduce the length of posting by the beginning of the next posting offset in posting table. This is thanks to the sorted postings. + +## Risk + +### Unexpected memory usage + +Users care the most about surprising spikes in memory usage. Currently the Store Gateway caches the whole index-cache.json. While it's silly to do so for all blocks, this +will happen anyway if query will span over large number of blocks and series. This means that while baseline memory will be reduced, baseline vs request memory difference will be even more noticeable. + +This tradeoff is acceptable, due to total memory used for all operation should be much smaller. Additionally such query spanning all of the blocks and series are unlikely and should be blocked by simple `sample` limit. + +### Benchmark + +How to micro-benchmark such change? mmaping is outside of Go run time, which counts allocations etc. + +### Do we need to mmap? + +mmap adds a lot of complexity and confusion especially around monitoring it's memory usage as it does not appear on Go profiles. + +While mmap is great for random access against a big file, in fact the current implementation of the [FileReader](https://github.com/prometheus/prometheus/blob/de0a772b8e7d27dc744810a1a693d97be027049a/tsdb/index/index.go#L664) +reallocates symbols, offsets and label name=value pairs while reading. This really defies the purpose of mmap as we want to combine all info in a dense few sequential sections of index-header binary format, +this file will be mostly read sequentially. Still, label values can be accessed randomly, that's why we propose to start with mmap straight away. + +### Is initial long startup, really a problem? + +After initial startup with persistent disk, next startup should be quick due to cached files on disk for old blocks. +Only new one will be iterated on. However still initial startup and adhoc syncs can be problematic for example: auto scaling. +To adapt to high load you want component to start up quickly. + +### LabelNames and LabelValues. + +Currently all of those methods are getting labels values and label names across all blocks in the system. + +This will load **all** blocks to the system on every such call. + +We have couple of options: + +* Limit with time range: https://github.com/thanos-io/thanos/issues/1811 +* Cache e.g popular label values queries like `__name__` + +## Alternatives + +* Introduce faster JSON parser like [json-iterator/go](https://github.com/json-iterator/go) + * However, this does not contribute towards faster creation of such `index-header` from TSDB index. +* Design external memory compatible index instead of TSDB index. + * This is being discussed in another threads. While we should look on this ASAP, it's rather a long term plan. +* Build `index-cache.json` on sidecar + * This unfortunately may require some extra memory for sidecar which we want to avoid. + +## Work Plan + +Replace `index-cache.json` with `index-header`: + * Implement building `index-header` file from pieces of TSDB index. + * Allow Store Gateway to build it instead of `index-cache.json` to disk. + * Allow Store Gateway to use `index-header` instead of `index-cache.json` for queries. + * Load those files using mmap + * Remove building `index-cache.json` in compactor. + +## Future Work + +* Load on demand at query time: + +We can maintain a pool with limited number of `index-header` files loaded at the time in Store Gateway. +With an LRU logic we should be able to deduce which of blocks should be unloaded and left on the disk. + +This proposes following algorithm: + +* blocks newer then X (e.g 2w) being always loaded +* blocks older than X will be loaded on demand on query time and kept cached until evicted. +* background eviction process unloads blocks without pending reads to the amount of Y (e.g 100 blocks) in LRU fashion. + +Both X and Y being configurable. From UX perspective it would be nice to set configurable memory limit for loaded blocks. + +* Build on demand at query time: + * Allow Store Gateway to build `index-header` on demand from bucket at query time. + * Do not build `index-header` on startup at all, just lazy background job if needed. + +[index-cache-code]: https://github.com/thanos-io/thanos/blob/bd9aa1b4be3bb5d841cb7271c29d02ebb5eb5168/pkg/block/index.go#L40 +[toc]: https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md#toc +[symbols]: https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md#symbol-table +[labels-indices-table]: https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md#label-offset-table +[label-index]: https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md#label-index +[postings-offsets]: https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md#postings-offset-table + From d758432436718ff6cd0d9136bc53653f4ec5cf47 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 7 Jan 2020 15:17:02 +0000 Subject: [PATCH 147/257] Updated Prometheus deps to master around 2.15.2. (#1947) Signed-off-by: Bartlomiej Plotka --- CHANGELOG.md | 9 ++ go.mod | 3 +- go.sum | 9 +- pkg/block/index.go | 39 ++----- pkg/block/index_test.go | 83 ++++++++++++++ pkg/block/indexheader/json_reader.go | 71 ++++++++---- pkg/compact/compact_e2e_test.go | 8 +- pkg/compact/downsample/downsample_test.go | 17 ++- .../downsample/streamed_block_writer.go | 103 ++++-------------- 9 files changed, 194 insertions(+), 148 deletions(-) create mode 100644 pkg/block/index_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 05b3e5f400..134714a9c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Compactor now properly handles partial block uploads for all operation like rete * Removed `thanos_compact_sync_meta_*` metrics. Use `thanos_blocks_meta_*` metrics instead. * Added `thanos_consistency_delay_seconds` and `thanos_compactor_aborted_partial_uploads_deletion_attempts_total` metrics. + - [#1936](https://github.com/thanos-io/thanos/pull/1936) Store: Improved synchronization of meta JSON files. Store now properly handles corrupted disk cache. Added meta.json sync metrics. - [#1856](https://github.com/thanos-io/thanos/pull/1856) Receive: close DBReadOnly after flushing to fix a memory leak. - [#1882](https://github.com/thanos-io/thanos/pull/1882) Receive: upload to object storage as 'receive' rather than 'sidecar'. @@ -35,6 +36,14 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1881](https://github.com/thanos-io/thanos/pull/1881) Store Gateway: memcached support for index cache. See [documentation](docs/components/store.md/#index-cache) for further information. - [#1904](https://github.com/thanos-io/thanos/pull/1904) Add a skip-chunks option in Store Series API to improve the response time of `/api/v1/series` endpoint. +### Changed + +- [#1947](https://github.com/thanos-io/thanos/pull/1947) Upgraded Prometheus dependencies to v2.15.2. This includes: + + * Compactor: Significant reduction of memory footprint for compaction and downsampling process. + * Querier: Accepting spaces between time range and square bracket. e.g `[ 5m]` + * Querier: Improved PromQL parser performance. + ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 ### Added diff --git a/go.mod b/go.mod index bae87de512..aa141722b1 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible github.com/armon/go-metrics v0.3.0 - github.com/aws/aws-sdk-go v1.25.35 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/cespare/xxhash v1.1.0 @@ -71,7 +70,7 @@ require ( github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 github.com/prometheus/common v0.7.0 github.com/prometheus/procfs v0.0.6 // indirect - github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1 // Prometheus master v2.14.0 + github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef // master ~ v2.15.2 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/smartystreets/assertions v1.0.1 // indirect diff --git a/go.sum b/go.sum index 724168b406..640a65ba00 100644 --- a/go.sum +++ b/go.sum @@ -83,9 +83,8 @@ github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.23.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.25.35 h1:fe2tJnqty/v/50pyngKdNk/NP8PFphYDA1Z7N3EiiiE= -github.com/aws/aws-sdk-go v1.25.35/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= +github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -447,8 +446,8 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.6 h1:0qbH+Yqu/cj1ViVLvEWCP6qMQ4efWUj6bQqOEA0V0U4= github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= -github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1 h1:5ee1ewJCJYB7Bp314qaPcRNFaAPsdHN6BFzBC1wMVbQ= -github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1/go.mod h1:PVTKYlgELGIDbIKIyWRzD4WKjnavPynGOFLSuDpvOwU= +github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef h1:pYYKXo/zGx25kyViw+Gdbxd0ItIg+vkVKpwgWUEyIc4= +github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= diff --git a/pkg/block/index.go b/pkg/block/index.go index a86abb8045..0df14ded4d 100644 --- a/pkg/block/index.go +++ b/pkg/block/index.go @@ -1,6 +1,7 @@ package block import ( + "context" "fmt" "hash/crc32" "math/rand" @@ -282,7 +283,7 @@ func Repair(logger log.Logger, dir string, id ulid.ULID, source metadata.SourceT } defer runutil.CloseWithErrCapture(&err, chunkw, "repair chunk writer") - indexw, err := index.NewWriter(filepath.Join(resdir, IndexFilename)) + indexw, err := index.NewWriter(context.TODO(), filepath.Join(resdir, IndexFilename)) if err != nil { return resid, errors.Wrap(err, "open index writer") } @@ -406,17 +407,19 @@ func rewrite( meta *metadata.Meta, ignoreChkFns []ignoreFnType, ) error { - symbols, err := indexr.Symbols() - if err != nil { - return err + symbols := indexr.Symbols() + for symbols.Next() { + if err := indexw.AddSymbol(symbols.At()); err != nil { + return errors.Wrap(err, "add symbol") + } } - if err := indexw.AddSymbols(symbols); err != nil { - return err + if symbols.Err() != nil { + return errors.Wrap(symbols.Err(), "next symbol") } all, err := indexr.Postings(index.AllPostingsKey()) if err != nil { - return err + return errors.Wrap(err, "postings") } all = indexr.SortedPostings(all) @@ -434,7 +437,7 @@ func rewrite( id := all.At() if err := indexr.Series(id, &lset, &chks); err != nil { - return err + return errors.Wrap(err, "series") } // Make sure labels are in sorted order. sort.Sort(lset) @@ -442,7 +445,7 @@ func rewrite( for i, c := range chks { chks[i].Chunk, err = chunkr.Chunk(c.Ref) if err != nil { - return err + return errors.Wrap(err, "chunk read") } } @@ -513,24 +516,6 @@ func rewrite( i++ lastSet = s.lset } - - s := make([]string, 0, 256) - for n, v := range values { - s = s[:0] - - for x := range v { - s = append(s, x) - } - if err := indexw.WriteLabelIndex([]string{n}, s); err != nil { - return errors.Wrap(err, "write label index") - } - } - - for _, l := range postings.SortedKeys() { - if err := indexw.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value)); err != nil { - return errors.Wrap(err, "write postings") - } - } return nil } diff --git a/pkg/block/index_test.go b/pkg/block/index_test.go new file mode 100644 index 0000000000..257005c7e2 --- /dev/null +++ b/pkg/block/index_test.go @@ -0,0 +1,83 @@ +package block + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/go-kit/kit/log" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/tsdb/index" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestRewrite(t *testing.T) { + ctx := context.Background() + + tmpDir, err := ioutil.TempDir("", "test-indexheader") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() + + b, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + {{Name: "a", Value: "1"}}, + {{Name: "a", Value: "2"}}, + {{Name: "a", Value: "3"}}, + {{Name: "a", Value: "4"}}, + {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}}, + }, 150, 0, 1000, nil, 124) + testutil.Ok(t, err) + + ir, err := index.NewFileReader(filepath.Join(tmpDir, b.String(), IndexFilename)) + testutil.Ok(t, err) + + defer func() { testutil.Ok(t, ir.Close()) }() + + cr, err := chunks.NewDirReader(filepath.Join(tmpDir, b.String(), "chunks"), nil) + testutil.Ok(t, err) + + defer func() { testutil.Ok(t, cr.Close()) }() + + m := &metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ULID: ULID(1)}, + Thanos: metadata.Thanos{}, + } + + testutil.Ok(t, os.MkdirAll(filepath.Join(tmpDir, m.ULID.String()), os.ModePerm)) + iw, err := index.NewWriter(ctx, filepath.Join(tmpDir, m.ULID.String(), IndexFilename)) + testutil.Ok(t, err) + defer iw.Close() + + cw, err := chunks.NewWriter(filepath.Join(tmpDir, m.ULID.String())) + testutil.Ok(t, err) + + defer cw.Close() + + testutil.Ok(t, rewrite(log.NewNopLogger(), ir, cr, iw, cw, m, []ignoreFnType{func(mint, maxt int64, prev *chunks.Meta, curr *chunks.Meta) (bool, error) { + return curr.MaxTime == 696, nil + }})) + + testutil.Ok(t, iw.Close()) + testutil.Ok(t, cw.Close()) + + ir2, err := index.NewFileReader(filepath.Join(tmpDir, m.ULID.String(), IndexFilename)) + testutil.Ok(t, err) + + defer func() { testutil.Ok(t, ir2.Close()) }() + + all, err := ir2.Postings(index.AllPostingsKey()) + testutil.Ok(t, err) + + for p := ir2.SortedPostings(all); p.Next(); { + var lset labels.Labels + var chks []chunks.Meta + + testutil.Ok(t, ir2.Series(p.At(), &lset, &chks)) + testutil.Equals(t, 1, len(chks)) + } + +} diff --git a/pkg/block/indexheader/json_reader.go b/pkg/block/indexheader/json_reader.go index dd79858874..cf0277a929 100644 --- a/pkg/block/indexheader/json_reader.go +++ b/pkg/block/indexheader/json_reader.go @@ -3,6 +3,7 @@ package indexheader import ( "context" "encoding/json" + "hash/crc32" "io/ioutil" "os" "path/filepath" @@ -13,6 +14,7 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/tsdb/encoding" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block" @@ -56,6 +58,50 @@ func (b realByteSlice) Sub(start, end int) index.ByteSlice { return b[start:end] } +// The table gets initialized with sync.Once but may still cause a race +// with any other use of the crc32 package anywhere. Thus we initialize it +// before. +var castagnoliTable *crc32.Table + +func init() { + castagnoliTable = crc32.MakeTable(crc32.Castagnoli) +} + +// readSymbols reads the symbol table fully into memory and allocates proper strings for them. +// Strings backed by the mmap'd memory would cause memory faults if applications keep using them +// after the reader is closed. +func readSymbols(bs index.ByteSlice, version int, off int) ([]string, map[uint32]string, error) { + if off == 0 { + return nil, nil, nil + } + d := encoding.NewDecbufAt(bs, off, castagnoliTable) + + var ( + origLen = d.Len() + cnt = d.Be32int() + basePos = uint32(off) + 4 + nextPos = basePos + uint32(origLen-d.Len()) + symbolSlice []string + symbols = map[uint32]string{} + ) + if version == index.FormatV2 { + symbolSlice = make([]string, 0, cnt) + } + + for d.Err() == nil && d.Len() > 0 && cnt > 0 { + s := d.UvarintStr() + + if version == index.FormatV2 { + symbolSlice = append(symbolSlice, s) + } else { + symbols[nextPos] = s + nextPos = basePos + uint32(origLen-d.Len()) + } + cnt-- + } + return symbolSlice, symbols, errors.Wrap(d.Err(), "read symbols") +} + func getSymbolTable(b index.ByteSlice) (map[uint32]string, error) { version := int(b.Range(4, 5)[0]) @@ -68,7 +114,7 @@ func getSymbolTable(b index.ByteSlice) (map[uint32]string, error) { return nil, errors.Wrap(err, "read TOC") } - symbolsV2, symbolsV1, err := index.ReadSymbols(b, version, int(toc.Symbols)) + symbolsV2, symbolsV1, err := readSymbols(b, version, int(toc.Symbols)) if err != nil { return nil, errors.Wrap(err, "read symbols") } @@ -120,32 +166,15 @@ func WriteJSON(logger log.Logger, indexFn string, fn string) error { } // Extract label value indices. - lnames, err := indexr.LabelIndices() + lnames, err := indexr.LabelNames() if err != nil { return errors.Wrap(err, "read label indices") } - for _, lns := range lnames { - if len(lns) != 1 { - continue - } - ln := lns[0] - - tpls, err := indexr.LabelValues(ln) + for _, ln := range lnames { + vals, err := indexr.LabelValues(ln) if err != nil { return errors.Wrap(err, "get label values") } - vals := make([]string, 0, tpls.Len()) - - for i := 0; i < tpls.Len(); i++ { - v, err := tpls.At(i) - if err != nil { - return errors.Wrap(err, "get label value") - } - if len(v) != 1 { - return errors.Errorf("unexpected tuple length %d", len(v)) - } - vals = append(vals, v[0]) - } v.LabelValues[ln] = vals } diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 9a7411b2e9..926175dc68 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -192,7 +192,7 @@ func TestGroup_Compact_e2e(t *testing.T) { numSamples: 100, mint: 0, maxt: 1000, extLset: extLabels, res: 124, series: []labels.Labels{ {{Name: "a", Value: "1"}}, - {{Name: "a", Value: "2"}, {Name: "a", Value: "2"}}, + {{Name: "a", Value: "2"}, {Name: "b", Value: "2"}}, {{Name: "a", Value: "3"}}, {{Name: "a", Value: "4"}}, }, @@ -247,7 +247,7 @@ func TestGroup_Compact_e2e(t *testing.T) { numSamples: 100, mint: 0, maxt: 1000, extLset: extLabels2, res: 124, series: []labels.Labels{ {{Name: "a", Value: "1"}}, - {{Name: "a", Value: "2"}, {Name: "a", Value: "2"}}, + {{Name: "a", Value: "2"}, {Name: "b", Value: "2"}}, {{Name: "a", Value: "3"}}, {{Name: "a", Value: "4"}}, }, @@ -371,7 +371,7 @@ func createAndUpload(t testing.TB, bkt objstore.Bucket, blocks []blockgenSpec) ( testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(prepareDir)) }() - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() for _, b := range blocks { @@ -407,7 +407,7 @@ func createEmptyBlock(dir string, mint int64, maxt int64, extLset labels.Labels, return ulid.ULID{}, errors.Wrap(err, "close index") } - w, err := index.NewWriter(path.Join(dir, uid.String(), "index")) + w, err := index.NewWriter(context.Background(), path.Join(dir, uid.String(), "index")) if err != nil { return ulid.ULID{}, errors.Wrap(err, "new index") } diff --git a/pkg/compact/downsample/downsample_test.go b/pkg/compact/downsample/downsample_test.go index 3e0de64351..9b13acf652 100644 --- a/pkg/compact/downsample/downsample_test.go +++ b/pkg/compact/downsample/downsample_test.go @@ -568,11 +568,11 @@ func (b *memBlock) Meta() tsdb.BlockMeta { return tsdb.BlockMeta{} } -func (b *memBlock) Postings(name, val string) (index.Postings, error) { +func (b *memBlock) Postings(name string, val ...string) (index.Postings, error) { allName, allVal := index.AllPostingsKey() - if name != allName || val != allVal { - return nil, errors.New("unsupported call to Postings()") + if name != allName || val[0] != allVal { + return nil, errors.New("unexpected call to Postings() that is not AllVall") } sort.Slice(b.postings, func(i, j int) bool { return labels.Compare(b.series[b.postings[i]].lset, b.series[b.postings[j]].lset) < 0 @@ -593,15 +593,20 @@ func (b *memBlock) Series(id uint64, lset *labels.Labels, chks *[]chunks.Meta) e } func (b *memBlock) Chunk(id uint64) (chunkenc.Chunk, error) { - if id >= uint64(b.numberOfChunks) { + if id >= b.numberOfChunks { return nil, errors.Wrapf(tsdb.ErrNotFound, "chunk with ID %d does not exist", id) } return b.chunks[id], nil } -func (b *memBlock) Symbols() (map[string]struct{}, error) { - return b.symbols, nil +func (b *memBlock) Symbols() index.StringIter { + res := make([]string, 0, len(b.symbols)) + for s := range b.symbols { + res = append(res, s) + } + sort.Strings(res) + return index.NewStringListIter(res) } func (b *memBlock) SortedPostings(p index.Postings) index.Postings { diff --git a/pkg/compact/downsample/streamed_block_writer.go b/pkg/compact/downsample/streamed_block_writer.go index f5b5b84184..bde6397869 100644 --- a/pkg/compact/downsample/streamed_block_writer.go +++ b/pkg/compact/downsample/streamed_block_writer.go @@ -1,6 +1,7 @@ package downsample import ( + "context" "io" "path/filepath" @@ -19,32 +20,6 @@ import ( "github.com/thanos-io/thanos/pkg/runutil" ) -type labelValues map[string]struct{} - -func (lv labelValues) add(value string) { - lv[value] = struct{}{} -} - -func (lv labelValues) get(set *[]string) { - for value := range lv { - *set = append(*set, value) - } -} - -type labelsValues map[string]labelValues - -func (lv labelsValues) add(labelSet labels.Labels) { - for _, label := range labelSet { - values, ok := lv[label.Name] - if !ok { - // Add new label. - values = labelValues{} - lv[label.Name] = values - } - values.add(label.Value) - } -} - // streamedBlockWriter writes downsampled blocks to a new data block. Implemented to save memory consumption // by writing chunks data right into the files, omitting keeping them in-memory. Index and meta data should be // sealed afterwards, when there aren't more series to process. @@ -62,9 +37,7 @@ type streamedBlockWriter struct { indexReader tsdb.IndexReader closers []io.Closer - labelsValues labelsValues // labelsValues list of used label sets: name -> []values. - memPostings *index.MemPostings // memPostings contains references from label name:value -> postings. - postings uint64 // postings is a current posting position. + seriesRefs uint64 // postings is a current posting position. } // NewStreamedBlockWriter returns streamedBlockWriter instance, it's not concurrency safe. @@ -100,32 +73,30 @@ func NewStreamedBlockWriter( } closers = append(closers, chunkWriter) - indexWriter, err := index.NewWriter(filepath.Join(blockDir, block.IndexFilename)) + indexWriter, err := index.NewWriter(context.TODO(), filepath.Join(blockDir, block.IndexFilename)) if err != nil { return nil, errors.Wrap(err, "open index writer in streamedBlockWriter") } closers = append(closers, indexWriter) - symbols, err := indexReader.Symbols() - if err != nil { - return nil, errors.Wrap(err, "read symbols") + symbols := indexReader.Symbols() + for symbols.Next() { + if err = indexWriter.AddSymbol(symbols.At()); err != nil { + return nil, errors.Wrap(err, "add symbols") + } } - - err = indexWriter.AddSymbols(symbols) - if err != nil { - return nil, errors.Wrap(err, "add symbols") + if err := symbols.Err(); err != nil { + return nil, errors.Wrap(err, "read symbols") } return &streamedBlockWriter{ - logger: logger, - blockDir: blockDir, - indexReader: indexReader, - indexWriter: indexWriter, - chunkWriter: chunkWriter, - meta: originMeta, - closers: closers, - labelsValues: make(labelsValues, 1024), - memPostings: index.NewUnorderedMemPostings(), + logger: logger, + blockDir: blockDir, + indexReader: indexReader, + indexWriter: indexWriter, + chunkWriter: chunkWriter, + meta: originMeta, + closers: closers, }, nil } @@ -146,14 +117,12 @@ func (w *streamedBlockWriter) WriteSeries(lset labels.Labels, chunks []chunks.Me return errors.Wrap(err, "add chunks") } - if err := w.indexWriter.AddSeries(w.postings, lset, chunks...); err != nil { + if err := w.indexWriter.AddSeries(w.seriesRefs, lset, chunks...); err != nil { w.ignoreFinalize = true return errors.Wrap(err, "add series") } - w.labelsValues.add(lset) - w.memPostings.Add(w.postings, lset) - w.postings++ + w.seriesRefs++ w.totalChunks += uint64(len(chunks)) for i := range chunks { @@ -183,14 +152,6 @@ func (w *streamedBlockWriter) Close() error { // Finalize saves prepared index and metadata to corresponding files. - if err := w.writeLabelSets(); err != nil { - return errors.Wrap(err, "write label sets") - } - - if err := w.writeMemPostings(); err != nil { - return errors.Wrap(err, "write mem postings") - } - for _, cl := range w.closers { merr.Add(cl.Close()) } @@ -243,37 +204,13 @@ func (w *streamedBlockWriter) syncDir() (err error) { return nil } -// writeLabelSets fills the index writer with label sets. -func (w *streamedBlockWriter) writeLabelSets() error { - s := make([]string, 0, 256) - for n, v := range w.labelsValues { - s = s[:0] - v.get(&s) - if err := w.indexWriter.WriteLabelIndex([]string{n}, s); err != nil { - return errors.Wrap(err, "write label index") - } - } - return nil -} - -// writeMemPostings fills the index writer with mem postings. -func (w *streamedBlockWriter) writeMemPostings() error { - w.memPostings.EnsureOrder() - for _, l := range w.memPostings.SortedKeys() { - if err := w.indexWriter.WritePostings(l.Name, l.Value, w.memPostings.Get(l.Name, l.Value)); err != nil { - return errors.Wrap(err, "write postings") - } - } - return nil -} - // writeMetaFile writes meta file. func (w *streamedBlockWriter) writeMetaFile() error { w.meta.Version = metadata.MetaVersion1 w.meta.Thanos.Source = metadata.CompactorSource w.meta.Stats.NumChunks = w.totalChunks w.meta.Stats.NumSamples = w.totalSamples - w.meta.Stats.NumSeries = w.postings + w.meta.Stats.NumSeries = w.seriesRefs return metadata.Write(w.logger, w.blockDir, &w.meta) } From 718e51ac6731dcacfd56c63d50b2b3d33148bd4c Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 7 Jan 2020 15:21:06 +0000 Subject: [PATCH 148/257] Fixed whitespaces on binary header docs; addressed missing comment. (#1948) Signed-off-by: Bartlomiej Plotka --- .../201912_thanos_binary_index_header.md | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/proposals/201912_thanos_binary_index_header.md b/docs/proposals/201912_thanos_binary_index_header.md index a1cef048b2..5f870f78c4 100644 --- a/docs/proposals/201912_thanos_binary_index_header.md +++ b/docs/proposals/201912_thanos_binary_index_header.md @@ -32,8 +32,8 @@ For each block that we want to serve metrics from, the Store Gateway has to have The `sync` process includes: * Download meta.json or fetch from disk if cached. -* Check if index-cache.json is present on the disk. If not: - * Check if index-cache.json is present in the bucket. If not: +* Check if index-cache.json is present on the disk. If not: + * Check if index-cache.json is present in the bucket. If not: * Download the whole TSDB index file, mmap all of it. * Build index-cache.json * else: @@ -46,26 +46,26 @@ The current, mentioned [index-cache.json][index-cache-code] holds block’s: * [Symbols][symbols] * LabelValues: * Calculated from all [LabelsIndicesTable][labels-indices-table], that each points to [LabelIndex][label-index] entry. - * Used for matching against request selectors and LabelValues/LabelNames calls. + * Used for matching against request selectors and LabelValues/LabelNames calls. * [Postings Offsets][postings-offsets] -There are few problems with this approach: +There are few problems with this approach: 1. Downloading the whole TSDB index file (which can be from few MB up to 64GB or even more when limit will be increased soon) is expensive and might be slow. 2. Building such index-cache requires at least 2x `index-cache.json` size of memory. 3. Parsing large index-cache.json files (which can be from few KB to few GB) might be slow. 4. We keep the full content of index-cache file in memory for all blocks. - * Some blocks are never queried. + * Some blocks are never queried. * Some posting offsets and label values within each block are never used. 1, 2 & 3 contributes to Store Gateway startup being slow and resource consuming: https://github.com/thanos-io/thanos/issues/448 4. causes unnecessary constant cache for unused blocks. -This design is trying to address those four problems. +This design is trying to address those four problems. ## Goals - + * Reduce confusion between index-cache.json and IndexCache for series and postings. * Use constant amount of memory for building index-header/index-cache; download only required pieces of index from bucket and compute new TOC file. * Leverage mmap and let OS unload unused pages from the physical memory as needed to satisfy the queries against the block. We can leverage mmap due to random access against label values and postings offsets. @@ -84,11 +84,11 @@ This design is trying to address those four problems. ## Proposal -TSDB index is in binary [format](https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md). +TSDB index is in binary [format](https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md). -To allow reduced resource consumption and effort when building (1), (2), "index-header" for blocks we plan to reuse similar format for sections like symbols, label indices and +To allow reduced resource consumption and effort when building (1), (2), "index-header" for blocks we plan to reuse similar format for sections like symbols, label indices and posting offsets in separate the file called `index-header` that will replace currently existing `index-cache.json`. - + The process for building this will be as follows: * Thanks to https://github.com/thanos-io/thanos/pull/1792 we can check final size of index and scan for TOC file. @@ -96,7 +96,7 @@ The process for building this will be as follows: * Get the symbols table and copy it to the local file. * Get the posting table and copy it to the local file. * Write new TOC file at the end of file. - + With that effort building time and resources should be compared with downloading the prebuilt `index-header` from the bucket. This allows us to reduce complexity of the system as we don't need to cache that in object storage by compactor. @@ -108,13 +108,13 @@ Given the minimal memory that each loaded block should take now, this is describ ### Length of Posting to fetch -While idea of combing different pieces of TSDB index as our index-header is great, unfortunately we heavily rely -on information about size of each posting represented as `postingRange.End`. +While idea of combing different pieces of TSDB index as our index-header is great, unfortunately we heavily rely +on information about size of each posting represented as `postingRange.End`. -We need to know that apriori to know how to partition and how many bytes we need to fetch from the storage to get each posting: https://github.com/thanos-io/thanos/blob/7e11afe64af0c096743a3de8a594616abf52be45/pkg/store/bucket.go#L1567 +We need to know apriori how to partition and how many bytes we need to fetch from the storage to get each posting: https://github.com/thanos-io/thanos/blob/7e11afe64af0c096743a3de8a594616abf52be45/pkg/store/bucket.go#L1567 To calculate those sizes we use [`indexr.PostingsRanges()`](https://github.com/thanos-io/thanos/blob/7e11afe64af0c096743a3de8a594616abf52be45/pkg/block/index.go#L156-L155) which -scans through `posting` section of the TSDB index. Having to fetch whole postings section just to get size of each posting +scans through `posting` section of the TSDB index. Having to fetch whole postings section just to get size of each posting makes this proposal less valuable as we still need to download big part of index and traverse through it instead of what we propose in [#Proposal](./201912_thanos_binary_index_header.md#proposal) For series we don't know the exact size either, however we estimate max size of each series to be 64*1024. It depends on sane number of label pairs and chunks per series. @@ -131,13 +131,13 @@ We have few options: However there is one that this proposal aims for: -* We can deduce the length of posting by the beginning of the next posting offset in posting table. This is thanks to the sorted postings. +* We can deduce the length of posting by the beginning of the next posting offset in posting table. This is thanks to the sorted postings. ## Risk ### Unexpected memory usage -Users care the most about surprising spikes in memory usage. Currently the Store Gateway caches the whole index-cache.json. While it's silly to do so for all blocks, this +Users care the most about surprising spikes in memory usage. Currently the Store Gateway caches the whole index-cache.json. While it's silly to do so for all blocks, this will happen anyway if query will span over large number of blocks and series. This means that while baseline memory will be reduced, baseline vs request memory difference will be even more noticeable. This tradeoff is acceptable, due to total memory used for all operation should be much smaller. Additionally such query spanning all of the blocks and series are unlikely and should be blocked by simple `sample` limit. @@ -145,14 +145,14 @@ This tradeoff is acceptable, due to total memory used for all operation should b ### Benchmark How to micro-benchmark such change? mmaping is outside of Go run time, which counts allocations etc. - + ### Do we need to mmap? mmap adds a lot of complexity and confusion especially around monitoring it's memory usage as it does not appear on Go profiles. While mmap is great for random access against a big file, in fact the current implementation of the [FileReader](https://github.com/prometheus/prometheus/blob/de0a772b8e7d27dc744810a1a693d97be027049a/tsdb/index/index.go#L664) -reallocates symbols, offsets and label name=value pairs while reading. This really defies the purpose of mmap as we want to combine all info in a dense few sequential sections of index-header binary format, -this file will be mostly read sequentially. Still, label values can be accessed randomly, that's why we propose to start with mmap straight away. +reallocates symbols, offsets and label name=value pairs while reading. This really defies the purpose of mmap as we want to combine all info in a dense few sequential sections of index-header binary format, +this file will be mostly read sequentially. Still, label values can be accessed randomly, that's why we propose to start with mmap straight away. ### Is initial long startup, really a problem? @@ -160,21 +160,21 @@ After initial startup with persistent disk, next startup should be quick due to Only new one will be iterated on. However still initial startup and adhoc syncs can be problematic for example: auto scaling. To adapt to high load you want component to start up quickly. -### LabelNames and LabelValues. +### LabelNames and LabelValues. Currently all of those methods are getting labels values and label names across all blocks in the system. This will load **all** blocks to the system on every such call. We have couple of options: - + * Limit with time range: https://github.com/thanos-io/thanos/issues/1811 * Cache e.g popular label values queries like `__name__` - + ## Alternatives * Introduce faster JSON parser like [json-iterator/go](https://github.com/json-iterator/go) - * However, this does not contribute towards faster creation of such `index-header` from TSDB index. + * However, this does not contribute towards faster creation of such `index-header` from TSDB index. * Design external memory compatible index instead of TSDB index. * This is being discussed in another threads. While we should look on this ASAP, it's rather a long term plan. * Build `index-cache.json` on sidecar @@ -182,7 +182,7 @@ We have couple of options: ## Work Plan -Replace `index-cache.json` with `index-header`: +Replace `index-cache.json` with `index-header`: * Implement building `index-header` file from pieces of TSDB index. * Allow Store Gateway to build it instead of `index-cache.json` to disk. * Allow Store Gateway to use `index-header` instead of `index-cache.json` for queries. @@ -194,15 +194,15 @@ Replace `index-cache.json` with `index-header`: * Load on demand at query time: We can maintain a pool with limited number of `index-header` files loaded at the time in Store Gateway. -With an LRU logic we should be able to deduce which of blocks should be unloaded and left on the disk. +With an LRU logic we should be able to deduce which of blocks should be unloaded and left on the disk. This proposes following algorithm: * blocks newer then X (e.g 2w) being always loaded * blocks older than X will be loaded on demand on query time and kept cached until evicted. -* background eviction process unloads blocks without pending reads to the amount of Y (e.g 100 blocks) in LRU fashion. - -Both X and Y being configurable. From UX perspective it would be nice to set configurable memory limit for loaded blocks. +* background eviction process unloads blocks without pending reads to the amount of Y (e.g 100 blocks) in LRU fashion. + +Both X and Y being configurable. From UX perspective it would be nice to set configurable memory limit for loaded blocks. * Build on demand at query time: * Allow Store Gateway to build `index-header` on demand from bucket at query time. From a532ff6932fe421acab2499eb64a54ec6e5aa3d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Tue, 7 Jan 2020 18:35:12 +0200 Subject: [PATCH 149/257] docs/release-propose: add myself as shepherd (#1949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- docs/release-process.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/release-process.md b/docs/release-process.md index 2a6bb52f49..b67959dcc4 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -28,9 +28,10 @@ Release shepherd responsibilities: * Perform releases (from first RC to actual release). * Announce all releases on all communication channels. -| Release | Time of first RC | Shepherd (Github handle) | +| Release | Time of first RC | Shepherd (GitHub handle) | |-----------|--------------------------|--------------------------| -| v0.10.0 | (planned) 8.01.2019 | TBD | +| v0.11.0 | (planned) 19.02.2020 | TBD | +| v0.10.0 | 8.01.2020 | `@GiedriusS` | | v0.9.0 | 26.11.2019 | `@bwplotka` | | v0.8.0 | 9.10.2019 | `@bwplotka` | | v0.7.0 | 28.08.2019 | `@domgreen` | From 9c8443583a4b5a0c245c8add4fa6ff7a71fc443d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Tue, 7 Jan 2020 22:26:48 +0200 Subject: [PATCH 150/257] store: fix a small house-keeping data race (#1951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `stats` is being accessed from the goroutines inside this function so it needs to be protected here in the same way with `mtx`. Move the house keeping out of the inner loop into the outer one to reduce the number of times we will need to lock/unlock the mutex. Signed-off-by: Giedrius Statkevičius --- pkg/store/bucket.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index d069d9c2ef..835147a32d 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -824,12 +824,15 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie } blocks := bs.getFor(req.MinTime, req.MaxTime, req.MaxResolutionWindow) + mtx.Lock() + stats.blocksQueried += len(blocks) + mtx.Unlock() + if s.debugLogging { debugFoundBlockSetOverview(s.logger, req.MinTime, req.MaxTime, req.MaxResolutionWindow, bs.labels, blocks) } for _, b := range blocks { - stats.blocksQueried++ b := b From c1d3dd025dd06ca9096ca9c55522cfd1668d3b77 Mon Sep 17 00:00:00 2001 From: johncming Date: Wed, 8 Jan 2020 17:07:58 +0800 Subject: [PATCH 151/257] rule: fix bug that ignore deduplicate addrs. (#1956) Signed-off-by: johncming --- cmd/thanos/rule.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 202aa8478a..a4e955e25b 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -742,7 +742,7 @@ func queryFunc( // TODO(bwplotka): Consider generating addresses in *url.URL. addrs := dnsProvider.Addresses() - removeDuplicateQueryAddrs(logger, duplicatedQuery, addrs) + addrs = removeDuplicateQueryAddrs(logger, duplicatedQuery, addrs) for _, i := range rand.Perm(len(addrs)) { u, err := url.Parse(fmt.Sprintf("http://%s", addrs[i])) From adb869bf455cb354c5db5394d87b459dc8ca703f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 8 Jan 2020 14:17:58 +0200 Subject: [PATCH 152/257] README: update before v0.10.0-rc.0 (#1959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 134714a9c6..bb3a855aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,10 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1882](https://github.com/thanos-io/thanos/pull/1882) Receive: upload to object storage as 'receive' rather than 'sidecar'. - [#1907](https://github.com/thanos-io/thanos/pull/1907) Store: Fixed the duration unit for the metric `thanos_bucket_store_series_gate_duration_seconds`. - [#1931](https://github.com/thanos-io/thanos/pull/1931) Compact: Fixed the compactor successfully exiting when actually an error occurred while compacting a blocks group. +- [#1872](https://github.com/thanos-io/thanos/pull/1872) Ruler: `/api/v1/rules` now shows a properly formatted value +- [#1888](https://github.com/thanos-io/thanos/pull/1888) Query: some typos were fixed in metric descriptions +- [#1945](https://github.com/thanos-io/thanos/pull/1945) `master` container images are now built with Go 1.13 +- [#1956](https://github.com/thanos-io/thanos/pull/1956) Ruler: now properly ignores duplicated query addresses ### Added @@ -35,6 +39,8 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add a new `--alertmanagers.sd-dns-interval` CLI option to specify the interval between DNS resolutions of Alertmanager hosts. - [#1881](https://github.com/thanos-io/thanos/pull/1881) Store Gateway: memcached support for index cache. See [documentation](docs/components/store.md/#index-cache) for further information. - [#1904](https://github.com/thanos-io/thanos/pull/1904) Add a skip-chunks option in Store Series API to improve the response time of `/api/v1/series` endpoint. +- [#1902](https://github.com/thanos-io/thanos/pull/1902) Sidecar: learned about a new option `--shipper.ignore-unequal-block-size`. If specified then the check that the min block time is equal to the max block time will be ignored. Only use this if you want to keep a long retention time and compaction on your Prometheus instance. +- [#1910](https://github.com/thanos-io/thanos/pull/1910) Query: `/api/v1/labels` now understands `POST` - useful for sending bigger requests ### Changed @@ -44,6 +50,10 @@ Compactor now properly handles partial block uploads for all operation like rete * Querier: Accepting spaces between time range and square bracket. e.g `[ 5m]` * Querier: Improved PromQL parser performance. +- [#1833](https://github.com/thanos-io/thanos/pull/1833) `--shipper.upload-compacted` flag has been promoted to non hidden, non experimental state. More info available [here](docs/quick-tutorial.md#uploading-old-metrics). +- [#1867](https://github.com/thanos-io/thanos/pull/1867) Ruler: now sets a `Thanos/$version` `User-Agent` in requests +- [#1887](https://github.com/thanos-io/thanos/pull/1887) Service discovery now deduplicates targets between different target groups + ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 ### Added From 31500460f1dc71fca9e155b6d986be7593c10de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 8 Jan 2020 16:09:29 +0200 Subject: [PATCH 153/257] CHANGELOG: update again (#1960) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb3a855aca..4a43177fcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,6 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1907](https://github.com/thanos-io/thanos/pull/1907) Store: Fixed the duration unit for the metric `thanos_bucket_store_series_gate_duration_seconds`. - [#1931](https://github.com/thanos-io/thanos/pull/1931) Compact: Fixed the compactor successfully exiting when actually an error occurred while compacting a blocks group. - [#1872](https://github.com/thanos-io/thanos/pull/1872) Ruler: `/api/v1/rules` now shows a properly formatted value -- [#1888](https://github.com/thanos-io/thanos/pull/1888) Query: some typos were fixed in metric descriptions - [#1945](https://github.com/thanos-io/thanos/pull/1945) `master` container images are now built with Go 1.13 - [#1956](https://github.com/thanos-io/thanos/pull/1956) Ruler: now properly ignores duplicated query addresses @@ -39,7 +38,6 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add a new `--alertmanagers.sd-dns-interval` CLI option to specify the interval between DNS resolutions of Alertmanager hosts. - [#1881](https://github.com/thanos-io/thanos/pull/1881) Store Gateway: memcached support for index cache. See [documentation](docs/components/store.md/#index-cache) for further information. - [#1904](https://github.com/thanos-io/thanos/pull/1904) Add a skip-chunks option in Store Series API to improve the response time of `/api/v1/series` endpoint. -- [#1902](https://github.com/thanos-io/thanos/pull/1902) Sidecar: learned about a new option `--shipper.ignore-unequal-block-size`. If specified then the check that the min block time is equal to the max block time will be ignored. Only use this if you want to keep a long retention time and compaction on your Prometheus instance. - [#1910](https://github.com/thanos-io/thanos/pull/1910) Query: `/api/v1/labels` now understands `POST` - useful for sending bigger requests ### Changed From 10d7ba4a18ac9c748cfe0a7468dbc2e1bdf7fc5d Mon Sep 17 00:00:00 2001 From: Paul Traylor Date: Wed, 8 Jan 2020 23:10:38 +0900 Subject: [PATCH 154/257] Use RFC 3339 Date to be more readable (#1954) Signed-off-by: Paul Traylor --- docs/release-process.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/release-process.md b/docs/release-process.md index b67959dcc4..fdf488b88b 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -30,13 +30,13 @@ Release shepherd responsibilities: | Release | Time of first RC | Shepherd (GitHub handle) | |-----------|--------------------------|--------------------------| -| v0.11.0 | (planned) 19.02.2020 | TBD | -| v0.10.0 | 8.01.2020 | `@GiedriusS` | -| v0.9.0 | 26.11.2019 | `@bwplotka` | -| v0.8.0 | 9.10.2019 | `@bwplotka` | -| v0.7.0 | 28.08.2019 | `@domgreen` | -| v0.6.0 | 12.07.2019 | `@GiedriusS` | -| v0.5.0 | 31.06.2019 | `@bwplotka` | +| v0.11.0 | (planned) 2020.02.19 | TBD | +| v0.10.0 | 2020.01.08 | `@GiedriusS` | +| v0.9.0 | 2019.11.26 | `@bwplotka` | +| v0.8.0 | 2019.10.09 | `@bwplotka` | +| v0.7.0 | 2019.08.28 | `@domgreen` | +| v0.6.0 | 2019.07.12 | `@GiedriusS` | +| v0.5.0 | 2019.06.31 | `@bwplotka` | # For maintainers: Cutting individual release From bfd65e2d0f48a5284c0b7ef88b39bd1cce270406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 8 Jan 2020 16:33:49 +0200 Subject: [PATCH 155/257] Cut release 0.10.0-rc.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the `CHANGELOG.md` to have a proper header, the `VERSION` file, version hashes in `tutorials/`. Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 2 +- VERSION | 2 +- tutorials/katacoda/thanos/1-globalview/courseBase.sh | 2 +- tutorials/katacoda/thanos/1-globalview/step2.md | 8 ++++---- tutorials/katacoda/thanos/1-globalview/step3.md | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a43177fcf..4d8dc8e8dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.) -## Unreleased +## [v0.10.0-rc.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0-rc.0) - 2020.01.08 ### Fixed diff --git a/VERSION b/VERSION index c70836ca5c..c8e421089d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.0-dev +0.10.0-rc.0 diff --git a/tutorials/katacoda/thanos/1-globalview/courseBase.sh b/tutorials/katacoda/thanos/1-globalview/courseBase.sh index 196c8b5b9f..dfa288e0d8 100644 --- a/tutorials/katacoda/thanos/1-globalview/courseBase.sh +++ b/tutorials/katacoda/thanos/1-globalview/courseBase.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash docker pull quay.io/prometheus/prometheus:v2.14.0 -docker pull quay.io/thanos/thanos:v0.9.0-rc.0 +docker pull quay.io/thanos/thanos:v0.10.0-rc.0 diff --git a/tutorials/katacoda/thanos/1-globalview/step2.md b/tutorials/katacoda/thanos/1-globalview/step2.md index a5ee9470b7..6cb687d3c1 100644 --- a/tutorials/katacoda/thanos/1-globalview/step2.md +++ b/tutorials/katacoda/thanos/1-globalview/step2.md @@ -10,7 +10,7 @@ component and can be invoked in a single command. Let's take a look at all the Thanos commands: ``` -docker run --rm quay.io/thanos/thanos:v0.9.0-rc.0 --help +docker run --rm quay.io/thanos/thanos:v0.10.0-rc.0 --help ```{{execute}} You should see multiple commands that solves different purposes. @@ -53,7 +53,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_eu1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-eu1 \ -u root \ - quay.io/thanos/thanos:v0.9.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0-rc.0 \ sidecar \ --http-address 0.0.0.0:19090 \ --grpc-address 0.0.0.0:19190 \ @@ -68,7 +68,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.9.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0-rc.0 \ sidecar \ --http-address 0.0.0.0:19091 \ --grpc-address 0.0.0.0:19191 \ @@ -81,7 +81,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus1_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-1-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.9.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0-rc.0 \ sidecar \ --http-address 0.0.0.0:19092 \ --grpc-address 0.0.0.0:19192 \ diff --git a/tutorials/katacoda/thanos/1-globalview/step3.md b/tutorials/katacoda/thanos/1-globalview/step3.md index 80d8066ec3..bbdbb05981 100644 --- a/tutorials/katacoda/thanos/1-globalview/step3.md +++ b/tutorials/katacoda/thanos/1-globalview/step3.md @@ -28,7 +28,7 @@ Click below snippet to start the Querier. ``` docker run -d --net=host --rm \ --name querier \ - quay.io/thanos/thanos:v0.9.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0-rc.0 \ query \ --http-address 0.0.0.0:29090 \ --query.replica-label replica \ From 6f7c3222faf3c56a425a2fff4650694768da4d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Wed, 8 Jan 2020 16:32:36 +0100 Subject: [PATCH 156/257] pkg/receive: simplify hashring file (#1895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit simplifies the receive/hashring.go file a bit. Most significantly it changes two sequential `if`s that can never both be true, to an `if/else`. Signed-off-by: Lucas Servén Marín --- pkg/receive/hashring.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/receive/hashring.go b/pkg/receive/hashring.go index 8332847aa3..b755756a2b 100644 --- a/pkg/receive/hashring.go +++ b/pkg/receive/hashring.go @@ -7,9 +7,8 @@ import ( "sort" "sync" - "github.com/prometheus/prometheus/prompb" - "github.com/cespare/xxhash" + "github.com/prometheus/prometheus/prompb" ) const sep = '\xff' @@ -120,12 +119,9 @@ func (m *multiHashring) GetN(tenant string, ts *prompb.TimeSeries, n uint64) (st // considered a default hashring and matches everything. if t == nil { found = true - } - if _, ok := t[tenant]; ok { + } else if _, ok := t[tenant]; ok { found = true } - // If the hashring has no tenants, then it is - // considered a default hashring and matches everything. if found { m.mu.Lock() m.cache[tenant] = m.hashrings[i] From 36f75364b472d159b8db372890308b397e090f01 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Wed, 8 Jan 2020 10:34:48 -0500 Subject: [PATCH 157/257] add wal compression option in rule and receive (#1933) Signed-off-by: yeya24 --- CHANGELOG.md | 1 + cmd/thanos/receive.go | 29 ++++++++++++++--------------- cmd/thanos/rule.go | 3 +++ docs/components/rule.md | 1 + 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a43177fcf..26b1334c06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1833](https://github.com/thanos-io/thanos/pull/1833) `--shipper.upload-compacted` flag has been promoted to non hidden, non experimental state. More info available [here](docs/quick-tutorial.md#uploading-old-metrics). - [#1867](https://github.com/thanos-io/thanos/pull/1867) Ruler: now sets a `Thanos/$version` `User-Agent` in requests - [#1887](https://github.com/thanos-io/thanos/pull/1887) Service discovery now deduplicates targets between different target groups +- [#1933](https://github.com/thanos-io/thanos/pull/1933) Add a flag `--tsdb.wal-compression` to configure whether to enable tsdb wal compression in ruler and receiver. ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 2ea0c962c6..f7f412800e 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -13,7 +13,6 @@ import ( opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage/tsdb" "github.com/thanos-io/thanos/pkg/block/metadata" @@ -73,6 +72,8 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { tsdbBlockDuration := modelDuration(cmd.Flag("tsdb.block-duration", "Duration for local TSDB blocks").Default("2h").Hidden()) + walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool() + m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { lset, err := parseFlagLabels(*labelStrs) if err != nil { @@ -87,6 +88,14 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { } } + tsdbOpts := &tsdb.Options{ + MinBlockDuration: *tsdbBlockDuration, + MaxBlockDuration: *tsdbBlockDuration, + RetentionDuration: *retention, + NoLockfile: true, + WALCompression: *walCompression, + } + // Local is empty, so try to generate a local endpoint // based on the hostname and the listening port. if *local == "" { @@ -121,14 +130,13 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { *rwClientServerName, *dataDir, objStoreConfig, + tsdbOpts, lset, - *retention, cw, *local, *tenantHeader, *replicaHeader, *replicationFactor, - *tsdbBlockDuration, comp, ) } @@ -156,27 +164,18 @@ func runReceive( rwClientServerName string, dataDir string, objStoreConfig *extflag.PathOrContent, + tsdbOpts *tsdb.Options, lset labels.Labels, - retention model.Duration, cw *receive.ConfigWatcher, endpoint string, tenantHeader string, replicaHeader string, replicationFactor uint64, - tsdbBlockDuration model.Duration, comp component.SourceStoreAPI, ) error { logger = log.With(logger, "component", "receive") level.Warn(logger).Log("msg", "setting up receive; the Thanos receive component is EXPERIMENTAL, it may break significantly without notice") - tsdbCfg := &tsdb.Options{ - RetentionDuration: retention, - NoLockfile: true, - MinBlockDuration: tsdbBlockDuration, - MaxBlockDuration: tsdbBlockDuration, - WALCompression: true, - } - localStorage := &tsdb.ReadyStorage{} rwTLSConfig, err := tls.NewServerConfig(log.With(logger, "protocol", "HTTP"), rwServerCert, rwServerKey, rwServerClientCA) if err != nil { @@ -225,7 +224,7 @@ func runReceive( { // TSDB. cancel := make(chan struct{}) - startTimeMargin := int64(2 * time.Duration(tsdbCfg.MinBlockDuration).Seconds() * 1000) + startTimeMargin := int64(2 * time.Duration(tsdbOpts.MinBlockDuration).Seconds() * 1000) g.Add(func() error { defer close(dbReady) defer close(uploadC) @@ -237,7 +236,7 @@ func runReceive( dataDir, log.With(logger, "component", "tsdb"), reg, - tsdbCfg, + tsdbOpts, ) // Before quitting, ensure the WAL is flushed and the DB is closed. diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index a4e955e25b..c9784b892b 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -79,6 +79,8 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { tsdbRetention := modelDuration(cmd.Flag("tsdb.retention", "Block retention time on local disk."). Default("48h")) + walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool() + alertmgrs := cmd.Flag("alertmanagers.url", "Alertmanager replica URLs to push firing alerts. Ruler claims success if push to at least one alertmanager from discovered succeeds. The scheme should not be empty e.g `http` might be used. The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Alertmanager IPs through respective DNS lookups. The port defaults to 9093 or the SRV record's value. The URL path is used as a prefix for the regular Alertmanager API path."). Strings() alertmgrsTimeout := cmd.Flag("alertmanagers.send-timeout", "Timeout for sending alerts to Alertmanager").Default("10s").Duration() @@ -126,6 +128,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { MaxBlockDuration: *tsdbBlockDuration, RetentionDuration: *tsdbRetention, NoLockfile: true, + WALCompression: *walCompression, } lookupQueries := map[string]struct{}{} diff --git a/docs/components/rule.md b/docs/components/rule.md index ea6670c63e..093c17726c 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -281,6 +281,7 @@ Flags: --eval-interval=30s The default evaluation interval to use. --tsdb.block-duration=2h Block duration for TSDB block. --tsdb.retention=48h Block retention time on local disk. + --tsdb.wal-compression Compress the tsdb WAL. --alertmanagers.url=ALERTMANAGERS.URL ... Alertmanager replica URLs to push firing alerts. Ruler claims success if push to at From 34ae9f90d9dec35445c71714fcaf15451a3cbdd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Wed, 8 Jan 2020 18:10:56 +0100 Subject: [PATCH 158/257] cmd/thanos/rule.go: simplify deduplication (#1965) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit changes the querier address deduplication logic so that we only iterate over the input once, rather than twice. Signed-off-by: Lucas Servén Marín --- cmd/thanos/rule.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index c9784b892b..395a16ecc8 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -704,18 +704,16 @@ func labelsTSDBToProm(lset labels.Labels) (res labels.Labels) { func removeDuplicateQueryAddrs(logger log.Logger, duplicatedQueriers prometheus.Counter, addrs []string) []string { set := make(map[string]struct{}) + deduplicated := make([]string, 0, len(addrs)) for _, addr := range addrs { if _, ok := set[addr]; ok { level.Warn(logger).Log("msg", "duplicate query address is provided - %v", addr) duplicatedQueriers.Inc() + continue } + deduplicated = append(deduplicated, addr) set[addr] = struct{}{} } - - deduplicated := make([]string, 0, len(set)) - for key := range set { - deduplicated = append(deduplicated, key) - } return deduplicated } From 880c8f8de3b44ce227aac0c2fcf0b3bfb6c14317 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Wed, 8 Jan 2020 15:26:29 -0500 Subject: [PATCH 159/257] Add VEXXHOST to adopters (#1968) This commit adds VEXXHOST to the list of adopters with the logo included as well. Signed-off-by: Mohammed Naser --- website/data/adopters.yml | 3 +++ website/static/logos/vexxhost.png | Bin 0 -> 20468 bytes 2 files changed, 3 insertions(+) create mode 100644 website/static/logos/vexxhost.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index 5a41850880..c84633a9d5 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -69,3 +69,6 @@ adopters: - name: Tencent url: https://github.com/tkestack/tke logo: tencent.png +- name: VEXXHOST, Inc. + url: https://vexxhost.com + logo: vexxhost.png diff --git a/website/static/logos/vexxhost.png b/website/static/logos/vexxhost.png new file mode 100644 index 0000000000000000000000000000000000000000..56b6045335cd65530e04044a0fe7968709ad3166 GIT binary patch literal 20468 zcmYg%V{|4>v~_ITww~CwGqLSVY&&^k+fHUCwr!ge+ngBRym#Gi*8S1F&gx#h>r~gS z)7AT&iu|T1g$RcS2Lb|uC?hSd0s;cY^4B(lf%<#)XfVzM0fE4^78CpST}(>M!Pdc9 z)zR40T*}_gxbDiwO?D1+kV{Dq6t>dFmN#dI?%UL$YU{6aR3C$nm4Z%i0DjendA_-M@W^DE!`-0M7_t2WDP$dk68T4{+GpjcVejM%JuF2MK*0s4ecU;#E% zp{6n+z5Wk*Fis_Sk0#x1v3?)JwWZC^{Hhj7LQE1TVv~lW3iyb4(3^7g33pKM*F<8W zGaaHwlam$};X#0R>#pg_fQz7ithL1f!`?y!q`In2L^?|RsAZy)-=~ z1amMXBA`%G~80W8`fNsPFSV)=-W?&bbf)+Tc8I3U1HrA@u zj}Rm%R*WhP+$h6B3vB%3Uj94Y#>}a{?*5d|(NV*Ym)7Yfg8A*Q zFg;UrA%6xgFd-^j6126D83R_C$3!PqP%^J5euc+mf_ikb>(^CTTL}YxuH}!;7&eu% zY>7>t#*PA`FE%=PJ=K*;2BhY2@(J2ENd2z&VZ%DTQwA>O<~`ipcFpM*!4je$HKGW7a!r1Q=6M<<+3Hs`EFjd_bN6A(AQa zQTdx*Hq zD*BJQMG<6RLz#Pl9oXAD;nENLdX=+z(=9(R?FC-@bHpi354rhd6TBKuy0omblZXnO z@KO0VD^=jSoenQqO>pk&f?R>W6vwAr~k*|>F+_8%45dBCN93*p6zSbWkKPuUm%vT0iXr> zzUtt5@YjE11-H4DjD>;%2;E;B1_Tsn4Fd7k0{wg7{XPDMl>)H;e+7)C0Q~>kAZGvF zY#{=EfPe^t$cT%md4gW*L;K=OEC)2u%hfZ6g+i$Orh)xa!6QrJ7~VQNf5P$L|A5TR z4dd9W96#NsEC!#8AzQnE5L!&&*y!kiBC414Eg5A@qQ3F|L#i1gRoiW>KJ#N}?kYQz zi|s2n*X#ah3SSa3AhubZ_h<9LBx|^nflQ7hTCp+BBguQ(PCDez-Z8NlHxge&>QD6j z#bI2(VS)cJUNM41QB1Vdoi@4aXLrB- zBtT$qmW4u{ZbJ_H3PXQ`yj|Ie-;Wp3niF1e8~O3jS`EtEWsP5QrG z{@cQT-hXR_Ina;zL6cdFzrc9hXK_dagRyL7Rbae}!X(Jdovk#<)w9%u1U5OcId{wI zMW$z(#ZPfnK6Xv*k$j1`N;kHUR*iNQDCE7?Zp8*Rh<+GY)ziyqk$P@y6{IS+I?BIce;-C2jYLt^(CJAI zo7+b*aE}~HpSkyWfwaqe*&YnXq$=%fwMU-nuco*PC>!e93F~JlrAz(C8Gl_EGu?=} zp9D<-!_7y(3ax43fM}e6i8@zk)U%L;Vu7=t=K453fc9tDaHA2eC9YU%4sp|iD z3yv663Cki@VL@K58UTe5R+B!v)UXk;J$Lbn)IY_lL{pHz1#xydTZX%7Ymi-TtSwX( zPS^a;fmZ=AZB!FUp7Ul}GERIeTf!ii@@quP+-Us5~<35mOPTouTInk)IsAy`SS#TY$VMLblugVK~0M9rM9Ok z4i~~uM}{y65Usls)%`J6Uab1(J_eW!S;Avq6y|h0EjQB6%gyD&YUO-#k|^!Q^QK zLa~M1NeH1qp_ty5Af>!s&FBdspZo**w~<)I9aSiiy3|Bt5b!1baY~pBLd8Iljy{ih zy68N*KRqE7D%_}dV_xv3?;14Q$V%vcv)M7c1O=(!)Itt05 zQVm`hIa$Ck6kXK6{$POJqDbpR{3y8<(@Ok=%CJM5MnRh_t-1;y>9_8w@~Ku}D%#If zu@&6+?4Fo4iLL-XUbP{y}gEKzIEp@bV31Z_>v?XF7U{|1%<6lMt$T2d%z(!3DWTxv<@UE;>p zw0u;VH#>fX=DSOh*59C0asp|~Xa=+GJH2w=YuDu+2A3As)ba&n|Hb70#YsR33Kf+v zHSM|VJA8yu%<(5Hb~zK~mLw@MfF%muoIv5zk2NolBTW&{L8O#}+Z}?eng}^4^f8t$ zQS40%DzWU?fm`9hQ?WORD$1{**2YcpmOKi zjN~OsPq_iZ$E``fQqA*(cK-Fv?SFX^Br4pySY(n#D*ZoShjo|?8D*kdW$~a!KNh&+FCmh+#J8l)9!Wb2M$}3*v7;`6NaW! zhKn4}Tv~i_Msvk^-rkTAFc!>3e(O>LpXqIrk$$Vv#*@(eKhc)wFEGLaL%=WzQPbie z0BQoX6QaL~R^ye;Mu5;QeQzG#8lUT=e$C*q5_b?KbXe4p2NQ${-XDKxaKTC*K>_t2 z7Yb0MVS_UI+AZZc@{Ww`}WG2VqKXp3nMM<=_s%g|!@BRPNfm{MSi-=}u5 zf3+yb)*Nx7Ng!n5&WO(6*8!-$yk+9AXjyBTG{q`9$s_~0KG9f*LNrC-Rj!kBRh9om z_)O2DwNu(SaA3@cH|+7Oe{@|2Fr-{gw61(MqIGnPg3%q=Tf98<(m7~{Ac;?wKq`bpRHNv*9AvC5!Fy%jrr ztho!@7R^l0oOzyLN46gU;jz)47=s2cK2+~<(N=NMz*OFcKD!kYOS-lv)2XMFM_;dX zCp)qtS4%6tVN&>EBu(fxS(yOkRNMl9W;+N*&Kl*H>mq@Se24rvo1u7!VVsx>c3B2t z_m6h3XR7b!r?Z?NgD*i^XSo3Q8Oo}8?I#F~<^?k^BlJMzLtc{DSF|&FMUMbbx46@M zFfS+X(zQ*Jy#sYNz0=CgY$rtEc9L4fV25Yn(57PwLcC|A45vXf`lUlu3P*6!IaaqNI=rxz5G? z0=y&pt4zK1J)x3(iQI)Nj?RM_``g1 zTdx|vAq*&;yE!RFv4cGsJ4ii`0G} z4)e97l!utP%%-SoBe7=cVjcxV_6>j0^(|P-3gL85V)k?-(!|j{1-+1>%aa(29OQqOpnOTx=i4nlw2a^ptu zEym&CQTRh#ESa%QbO>4&WmG#`AtNFwdP@T+Bf4R(Ld@;WqJ=|Mj!#6WSD!+K0QoWn zuH%%BgpgWcX@o)v*_JPtB*vD--%8%M@{Wf|?M1 zPDL^cgJN-ZB>i&hB2@%HP~Wru?o8yBrWS=Je|r?0zX`(0Kyg^)iPy9-|~(Z$Vwa!G0sW1<{R0r$}JXAU_)M zs(vY_PEaKP+F(q2@*2)~Uj}*STF}nKmQdU_M z|F%0XA$TJv7$520E%bV%G}pb=yB+EZiDMlXGaPs6Ag67IV1ONY*hs7KNyDyv$C+I^2^XXgic(xhv&4KX)ZDh*nPwwW zc;<(Tmjh&W>1^t3wD}Ie9YL`)cc(^nwFT`9zn?+7H6J8q1Z?#%D0iy$4?UjJa`ww2 zT(n(pz9`5F*nufR&u;e`60pY_3)c5($nnsb+i zhQ&O+K+YS4fMHL&cIXFe_H+m*F~@4)ST%bR(}g3#R==h3fOFrsJvgco!Vw)!qJ4HX z33L>FH4{)x0F7gpas2?9@y-Hx=CdM@$Y0*eE7}>*DbBx0^bK*);&5GE_sNW zQuFSZ(z4j~eRLQE%5ue$VCe~$rDoG(Qzef(HlBB7Yt-jnR&|wup z9($uTv7}vJ;n)_>P+{K+B+(gRk;6j&WnPtuWGvd7lkNwC{*Wmuh+V!V}(z%o*?D9%@5@TFhu0Jf>5b2#Sb>G>F_g zM-C{8ss~Wcr@?sVoa^_9jMQ8f)oT;EJGGKvq*9fKD5x2*#MeUj;fSXFCP|sA8?d@I zW0z;~Wur6-pz}uB#%x>Ou{Z8x*u|_@>3<; zw<+p97ie)V-;`f5rXcu50rUI2yzFV{VXtBmSqcHI!+G`AT!U@msp12;s?Mh|_Y>A)b zhBaOn_|#14*x-yL0RwSt;G;RL=VfjL@fkH0gWdC&%kNI$v652{LBO5AZa<0}0mPkH z#eTF27$9=~T87juOJYSS;z#VZ5LaQUN6^3N}RK zTuggg_|$#z!5(x(V>#_C6a`;3?ovFew1AM?YaprjJ3eA=O+!r%^yiNKJ8ElpAn57y ziAnGgja!QR+nU0Y6!7p4*Im^#@#$LO?ir6$9bUeFo~qbXE4F)Yt`WB+Awl^mKqZ1H zcKs~tOLH~zGL+MLdDl>NxALuWX^C5RoWzeTr0ke>l3*yPwho&*e%DfUgSWw_Oh;}d z_%PM`GV!^4OMDyktR@&z*ax>EapCoX7aJB_i(GnU*g?WtLEs|RUX+-mi=;Ac}{ zhG$&+4$`1v>gtb1{Igc%n{0%p9BkDNH+HNAYkU^5=AW46sG+!cks($;RszUk^ zBqClC>Eusm=ly+eh}$d0kqB8^O9v-F+Rus77xPRl1HBfYz47(LD%d^X=F=l3>@Z40 zS?4LDtWMSZT{#c_Tb_@Z@z|^pj5m`z^^lb_Nv|&is8@@bCx$G6F_+o_PBX-%J$5o{ z+s)VoZCZhRzbcg`_t_U1=KGQbY+IjFaKcNdwrAOck&wwSba%GBowY;jZw7vK@3t!U z7<)`~G^(Y$f^Dt&y-%+%SB|CK!=TIlL>z)V1gU;hf316;NA-HYJ z{_4s1J@)Tuz&L3xVwYh7(hQ94D2Edj>rNtUqNjEN*D5{7#4?Y_xCC3liou?At**g2?B8 zO3DGOPsS>|xvfWAYz|^V9S;_#gcVQ6$0zNxy+1qckK_!&@}yy87FeZ7y{ifyjP05i zR;)R~HVh+iV+o;P16uaEh-#3~zzI46Hz}!w1uv|R7=yv140z_wrsy~3TiFq4MP5og zPwQF00H1`BWrbcKs+?#JQ^>OU2#-NHX3zr!P7pz}#uxm@14fJh$4N(eX3?0{nHv6~ zc!PUwu0v3i0nHSChmmsiz>ro_90<85Tpz?K@-?TG=Hfl0w%}|@j^Hc=j`EB!cs&N@ z(vaFZGoGoHHCkuqz8TJ=cjj=OjT;ve=qQ({bQaTv;ckI*9nqd)(I;sfqXF>v%80yyZJ0a_A&ubQ1pDGi!cWhk%nKp^{WXNs#*H< zGvUH-N^H|KJEq05Hj|C27Gp)Hm4m0f)e_Biu7RH6lC|j38ma-dhR-1kdUI&xBY_aJQEJA;vju!x8O*k32@YeZQTC0jOoNHBcJqL@!ud-jAxT&{rkA;t?9h#`%%}eDrr-1Ly{8 zANYBT`Y52cn??%n@qNc9hNFaTwef8{S$P*OxC%Hwt+SX9QtdZ~eYEXcPibzV`9Nhj zdj@aQy!jBXjjd|z>RT%uoD$2Xrc0(wv_HII#FEY4A*XgO*m1Pvn$j(-G5o;6Hqcv4 zX1%cdsuo8vt{>@ZfRDj;yXE*Q=GRD%B9)sV~_|l~fNI{F9*|Tmqa=K_aK9CK0jml}K+tWAW-)zd^tJRj~sp zc{*a9`~}|mF7_@TiGb54rsbx^(BcFY(KW!62Ycr3)0$zTipCXNW4t&JV}@|ad(p;& zXyg)bz@KlOv)@VEoQyH!?cDRpw=yG$`4&PA(b((D9cBM4-(b7o{?@9FO?UWdEVR(U zG|Am8&!O+W&*dgk=_5Xl4pxwgnNNo}L54a3(l2)Cu9B9MRnYaOf>#y+;gbiv`IP5K z!K?d{)3}KbZCEejb2WbM@}lTEh6Rhk-(EPdeL}DA?6(fumqrkT^pmjU7H~!G5aeF> zO{`c)E28uQ@5mmw?RsKxk(+lMXf3Vk0+qu*lf@2V(P&55ZdcQVy3nm{Gx_3*@XKfV zI+$lcVvNsV7a)&C28Q44eR5MR<65(XbnecJ#b}!M9Y+`nr7b$UcC8O#8GBiGmPad& zDPzergw^-PIeFDzD1}l^Nuw#FgMmt3lcE~O1Yq3elmgQp^K1GiZ`sNkEO*_%ETv&A zMeKZR&8;tCN5QuIV@>BAmnlc5$D-I>Dee^E3_Rn*xydL#P84l97JJ{|nsPG`dmj&} zxSxFRZ%cC7LvvqHn1@AoGNHbh7-7d#{F%}7%-$Z)9^Z%t1I;+I1;ImU%%n%xj z#1ai;;mN}&gI6ye4M0SWs$fi3!ModlL(?_PD*d|8l5OA%BOm6dVMSQr)3AP&dTlkzHo>uV_UHzO;C^828W{(%0@7zyayX4g56T2ircv9y7{>+^nG2p7@yD zHg9{oh-0;j3WQvcAntF+2m2V%Wt!=fH5C-(jkik2vc zaL~oVgvE2`zYPGmQUvq`Mg&4t?|lFSZhemT>e|&NR0akr9)S!D_>BfjU&gEkR`B$? zzqcwYF(;mp33QTcvb~0odNAvGjrEu$k6FIA7Tc?8d$J=+!R5rTA4@E7@i%;TsSWc1 zf1|hxl@z~_W{n+R<#F6KVmb+>#F(&Xw?Lh+n5RS%|9I6**Tl&kpitC5B{Qmnbcc~) z8D=W|em}Pzp01B@g>znJ+`o>gh`htY<$$|qciEH<{Go9G?6KFLa)U+yHeL`dJ?X??X}!`pmF zpDAIeB#h@mLFM;#kS_&*TJ#rY;!wQZ=CKh@B1hT8(F4@0Y(Twqw;jrC(Nq6Mi}m*sziBLhdA ztji;=JNqRsD9M~e8(-8l^ctv}gP_tpOY6AvA)jDY&elkFB53AWX|^JJ#{*iNKIBM!fQp|I}zEALgM+oy*gX26V;eFc@> zDQa`lTsuCyq5guv)&HAasRpX^Q)+y#)c*+;E#L0f@hgvUJeN|+W`?H`}vTeVq9k~zzFuiQUS;x;q1rvjgu-DRj5Dc8WBIy&xp6@qE`+s5y&}FURTx zCr1@@m$Hn7?5hc-Krxu|3DiZvHj}-E0R6-9M_i_G3`yp`t-|Vr2Pi0Cm-I@K))AZW zcTpxl7__B{?r6x1+}iKqam3}Ax~HiOp}Y+aYp+dbDa_-v!nnFDG%Gwm|A3sTzGAyt z_3SVY!Uf`pdC`G}aNpsBUS*IrdjIj9p=F=^72u}$`^w2+cpvdlQqvC&SiTm;zQ@}P zf6_tVtq_+7gQvhlPBH$2*hx3snKwen>*id*34M=jOUlW99IH%+a~C$=gJ9<%lX_?1 zNU20GkD`fOD?5V>*?I6b%l-L&k%`}X5L(W;4;HEW5y?5`Mii(TTpj~b!Cy+mMGF0bOK<#OY) zzlmC^(caSVpT>1DWg~SLM6z+53jnduOUr)#=lAD2F242Gp-mfgP{NJ zxL=u?3%`XdDD|bm+wu&(zI!$2dK=>aSdGThP<|k+`w!sd>xfI8Y3Ija@x~4z31GBG02u1UCZueHg^& znC`{?(DB86&+DRDm(^G#DOb-&=D@g|xgCF`8J`*;i zNY}3N3bN^KsfvRAYJH}R@@t8M;*hhH{6bGhJZ>s^Qm6Vm1xD0w*_-A+{xdg zT3Cj2Ca5Popz9ei*GJk6T?Tmg#XuEpoe;4*_62=Glvhh(a;JFhSJ5J_7OpKM&?8cd z=duid#IPYJxPg$DUpK9ZOh>}ma?M21oM-Y~4Jx0cLjP;TOa^yP}mdit#y;K{YpooH@3J4IBHrT8Jld>j&#gdlZ54gN5C> z?HcbK_icyW%=;LTS{e3FKpFCA&7(&3U%@CB`${Cv_>00JkT#gJRXFb8#}G5=A!U?mJ}jaJQkhCw(mzf0sl8 z#!i33@;zIQmFS++r22N^Add76&8;MZS zs6wF`RF9s})-fwQmF8USkbKqH8(Vw2I>+lIno37`lleE|L+JP^NKO6R!0`I@cPYj9 zJ03KV1=t5AZ{(mgCw8&R0W37c@RxG@`#*0myt+pE(m>yVv!i=WRA<+nSTLD|3vKbc zwd~s6SVXJ#jPCKJeyMy6$+K6InarMQ4A@zrr3yrfZl!4^*9ncdW*M{Px?Q3qGYJaI zNe-e*NE&Jo+cZfb%8Bsoc)v2{A`b`5^D52PM`9dhUnNkC$}%y*!6sj-u3j9n!;Yhn z?68NcR^i$CEcV|esY;kbqEu!>3v81Y0-_j!A?mI(gJ8q$$ZOqp`QgmZhZ|oz#YI+|r6D zCfM41&ZuF7#K?Aias}QFV68Q?AptSvT$s%6+SLv~~#U5Y|I#s$`F1EqDEo0f6RDbckNQ*P1!ZBN$XOI;GA_ID=-$iDGB&=H2+b%fH zzL32HIUbyhV{Wc?|JkFE$jBAm?GYqy%X;go+v1n90N`%nlwciykQoq*0WMIs5 z^}8VO8c>Qzc;aI>>LP?wdkx&iFF)Cv7yTlZdK}mIMgoGUYlIHYML;r@CctdMrL4y5i_XuL5ihi&mJ!D08EFqLf5#n^oxQa zoDkjcbUcj)24fmQ@IzS-4Ao3`daVRT`YJHB+B2h5t?rN=&zj25Q&OaFoDId4N#Z5m za%FYk_52rtmr*MI=Em^5U1+mdpdQ%3C7 zp@55? zs3Bs1u(vt9m5C-yzL?6o(1H`>*L`sV=DCqm>4x zQG$;Z#~?$t%&RWpMVpn9XrfX`!469~_@V{qq;y*{@>P|wE zI=Ag2d8(&gnMKLWBEvZ5c_F4v$y=77m?*}!B>x~*tJ{fQy<};55bM!YvBK;nA!kG% zPR~(f&33#1^G%-$5q5!t$l9HqxDJ`VfqM6YQ3g~wpzWI?p4|^ElKGny?&eJ0X*n*- z;tz&w4rHFC718mxge}|!0Nf=U5r@s#Zpp&`9Bo6H##%GmqFv3kAkG>gz*&DeUkJAA z^QSMgnryP1C@<0)wJ2-1-~M5|yxZUQ@p?Q(K5G`Re18*U43#;$aei}3ZMgE27ZrV! z?5oK6E`?Zj7Ol#QqhzYm2MuoZ_Ue8eLvpS6?tXKXUgiIhtaqarPK~havJOi{6H?7{ z0tv&cJGuuNzy*~uX1ACyhwyEUs0>*3+BU8RZ^JgL3C*Rw&eJQK)*4Tcz+ zK^Y`e^y_3FEPTR0>a&ZXQ=%U6HTs#{L=Hc46Wa&MaTvC7a64*An0z^ zCUPqs%7dLkip_Jx%FRL$tVajKagoKflrJo^?kMbqeZnvCdHcpSQO9?C9}_#`3bqYf z68z9iW%^P2O0nF_M#>3lG(9F8faXSruxWRv!W7yv0;o;sE{7PDkob19Z$B+>+&<~Y2 zwH*i0!e7TX!bxpj&q_s@m#kJZej^n`CW@$GNNIWXM(>JwcD z#q{fYFqpiy^@n1XJ?aRKEs7Be>ShL%$nh9!7K!2TiTN_N(4)z^ObW7BzzX&Yb+V<) z+cO#CON9k9zi~G#;><4p-LUyWaUq*p~Bkz5D%Ee*gk? z43&92X|y?;uWEmSX1i@TgyH_d_rlVlR*H^SO+RxE8sYU8D)X~@C#s)%CU^GkLovw~ zs4fcma<)sAOpnZI`ABCsv(mfNi2KF3!}w9fa|7t`qkQk-!RGGeeSHhZEDYRX*N>5~ z(*i+k6%j-tmW!sS(4`cJ}d8uf}(E^{S2@cz!)%ndWe&t*qRkGi~WVHwQyK!#X{@Q`=twYrOO>!ngB6& zA%!?+RQeELKmyi=s=J-2g`M5qM}WoqZlJSDX|b(xp=UBDy(Nx5L$-iJvu(KNc}JuY z7PTR;6m~PpUFVc0pys#o8xTc#(<}z$J;RsO9L;=~BsN)!%;KSNvj%mof8t6O8Pkjy z@i^o=b7BwS)=7(G=1gy>pPl&H08s%54w&w|P*@_=VAYYck0}%igVT;V+{%rgi$dMP z>1@RXz!@0I13kfLqB(KfHS=}vHfK;9^g^e_n-i*!YI(GTj!6C5FC;mi74mn!e}WyK zfdu4bQjhATDc>CVZ-=08`qulp!g=1LLWWrlDW2DPI)=lba)r{#PG z^9}bBMy+AQda!-ec^@~3hBuTAutkEgcZo+@PT>?qnR-W zwc_zZLhjr*en8X8uVkp=Lv=+2N(+Xv*HDBTc98Q~eo;)*I>p!!ya)%y4!G?23- zcMe*e>oua`PNkWLmyXb8r)b0{gh5(6CNgrJgnfU*Y5_z(VI8D=0u*hke>pGC#C-hf zAoZ&b%W`g1SXiUCG9)+uThq|}KOzTdViHVHsip=ywg_mS>cZ-e-plx;3P`b^3|0A1 zLFb|iU@V`GRFUC;%*+{ZBDCnkA0`C+JY|^$UWT z2g+8~O?hB-8R>(r3N|nq=WxvUn~(+2g}a{25}oKTkdRq3eneS;=?@JwAvgVzisz)R zf|?PQJ=uy~2({n1nbCpxAIYoNN=MTn#`HFeZG{h8}nK^U(S=O~*j5hp*f2C5bvJ+~LAY-qoIy z-n_K!SF9>|-A z@hp@NnRF2L=<+j^fJ6jxp(e(7h;#&p4}8CNfpH}C zuWUq&w@?^lixWwRxo~TVPAnV>qN?N5c@kq;s3{(VT8wizgD=nKYcO>WP^QlP;z z;ShySjT{tn4*h6x?@v@e!3}?5;3dZ(GK#Ys!UOKAj}^XYroxCInRcak#X{(5)``K$ z0@ej2fI}>O-y5b}&20EaB(1hvs{ovBGj!#qLg2oV2yhDR@@_b(?e?vvTeB=Fb7=*_ z<wq*oS_ZuJF0LhO&ep&~i+BN4z$Dw4xSg58)N37lfrI4XG`Nj-!RUr6Wi#A!DAi~d@GmIl z?`ShTrP-c=(PAt9vFBp+I|}%arHB_ zF_oN<9wkc{ifJ`m4UxdWLsl4Uzqxl~7r2?6yg)ClgU`c3F#$V9|C3XXPshy5 zFx4N81drs9 zm3<4PY(&S}zKFTjoB5<<5VyKY0+hY0PBA6!G5Gw*$U zWVAB+TFm&&Y3efV+U{+qE8icNJCDcY#5(D|k*hv&MV*O})Y85+MQ!94uqw%As29~w zuYO=ik(=)gAW}5Ydn^Lg*bkvq1oaZ*bkqCXf?BtbP#jz_Z_`?OeRhg&RBMg58$ z(}nqt#ydfBBPnYmDe%B&B@D;0w&@i{8D`YBK{V`?noE+agE)!GY;;u4szrK5OjNGs z3&(52`2BNq@r)K6!n*}(0+~mg{9ZUPbA6p9K%LHO2jXSpCM%ys?(aXA-``?eQj6WT zlBZ@y2FeLQ%3Gnv(>wHGq4Uq#RYwZ~D3M`X`padcFd(7Ak56YmZN zf-WF>Hj{xsg69^*P#-6n+^@Is@sR7Io^$>ijl5Dh?#z(*G%Bl{yip(t+uV4I)ST34 z6f$+4;n>MXD%LBBW(W2=H2FeL5q%N1Hp~9lCMA+5#8bnZRocD>MJNt^9Cilx%0$WX zMPE{cV&>ZmEe_ud#&O9`@2F5AxQ?QPylW@#@@IbB3M}+XfygG+?-U2m=}Ud5AD4{< zO87m-ok<(N>u0*A7HSa)6+=?kid7>dH^gx&eNEHSAqeek?OMLa;vm2sL}8A?l^Drj z7C$8n(EhDdyUbr&UV9GG_T=}D-tft)R6?M(#30JZTRSmo1#j9MXAOyOyE6ag{cVI*eG-ZKTnKNlS{RfzB%ZxKZBg#^f?`Is z@uD!Q#5*T}e`k!mO{=hnP|TpWH2W+ZD3V<|H5VS2Z&REu>|z$wL<_%JVow&-vf(?? zK|2731Ov-b6cZBzESi9uRl=(7EIwWHUfvNwAuL4<_!^Si!WD#6%-c7;rWha&p44-l zf7+$Fk{XYi;5r?~(qZq-r6c&W67gXFRC6r6ajUOq5VNK%h46Ff@79}ov(aCOG#X|f1{dP#v8#|dK7 z*ZrhGwu2wZg`94<-B&-J%QUhFiPXexr%`^cjZ*OcGILtam@+|^f|HI%=_@H+5+iW; zJ^}}1)m1GN?tJB{&=0`OGD$2ST%yk)% zK=ei{f*kS_Fkmaq&HM270o=o9Ybal{Nl-=qkQ6sevhGfLm^^}61zga;hM=W+unuq? zLVN7T*WajFov1Q_^++69Ka31%+a2xFQ4oRMG=D-uWTMZOA!zfnWJ(Z!-lmXcAeh~P z_0*vxY>h5-4&gbMDX47#k>uGX%>8jgnI1Bz){8h!s$)&@xt z1MqQEm(f@sh{->$V_y1Dr+`Pha?)mIE>afUs<4=HryclP7}8**REhh<~cz!Z0(Z#kz2!|NaeKZken z%=II&HC071VS2Cibk2KwfpisA7!iK(q+Ej0biC?xtkceQ^6Y45NRWZN<3cx5-7Sl1 z)jJCefOu!~(smdoIAEVkf0ZH_w@f=n?Q~SB+eeYv#B2zJ-!C*bO{wk{p5=!}*?54M z?{w%aFgUu@v`qxyOEvXJMOHQ#@g4pNY2W97GDj*=;&mOf`%u@3;G^%C^S`V>GYPFTxMa|f)t!AsJ#w=BPi(mTto!8gj zA8_xv_nv$1J>T*9(BmSbArinA|KMJHmF%j4O}KEKxaSJ12Nj=mk(t z>E7@UVMtfk4f$bksEQ5ELV|UZz$y8$p5aVUI@!imK3rbaKWIf2NUdi@N922L+}?yv zgh@NYKYQ-nXyw_BlAg*dcDbTPYCBJR6mp^&@sQ#SZ!z*40SVyKg22iU+2o{?_Nrtn z{|2*ih-coBxv=p{W-M4cJ(QMxn^y-J=%GZ8sW>=onT=eStP{w4zNI$d5JnO_h6|qQ z*klbb%9pI24@SRfjb0~6ZrBs@d5qfx@tHkv_OdRqc{~w&5r?oHau7>54hjhiBps`; zZq4T5VK4*!G1Jnej;AQG^YHapCsbMW%UKE?<#_x~;PXsF8j_WU(Pt>lFWAK#!?q43 z4=qFJ4c_*LTN*$H4o6~W=Omf3+z9I>_oJhgd1#&MLx9+Xv%x`y-{)ZqCL(z7XNAn* zUe7y%WClpRp?b?wWYoE#$Eyg3+VUMxjBD}`!D10xko;q%b`T&6e0|i=#Pv}|h#~W^ z0n$wk6*2tfl*`^R|f%`nik6WD!QRa&eC1U7P2L#y@z}L5% z0zfqHIqy{#ntV_Xc1^Q~-7-ak z-z_y|Ko9VwNSAMLXt(C-mENJhV(!vPo32m%JeucIDc(N)DkxdcYExSyUq5|%&eFo2 zbKxJ#$@$hjO&d`)`F5LU`%+^3SgmuQ5>2YDAO0Xb-|Yy_a$tfoprWxk zcZ27Y&RxqH3Jm=TBaUt5kLZOf3~u9mZ6F$p880R50bkuuh3ZDa{r04kiM^}U7aMc) zMTI|3bSChZ&21%FC+-ZgY_R|Ng1E4FIJbLvv6>-Y&I;(hA ziEnJ4_w(uKEjn#Nbp#GTk8r+>KAUUGksX6Z+ax4(y1!fiCqoG&wN%F;9beViI`qCG zl*o}+zy0(bP3hz1P81fweQmAS#!m#nn@d-Tgw!M&?|cNK=19E;-Kv;1$>`bAz|gzO z(X_k7Z)c27*{3$yqgFM(-J>Ni(?Z!9Rky>;9ciH$mE|>E@owy%zK_Ta7~wlWPW=xm~DD(JOpcCo-+;kUIv6 zX-&U-KJ8x__R-dMyIcrbE(95xOHeqE7Bd?rI9T53_;T1GhoPkw7=`&BKp0g)0Nqn?)Dg&*$Ae0ffv2)n~Ql?X{dj z$zOp4`|%-rb|kp(Yc&|NcXxa-s8W$jVEmqr)HGs9P zq+u4pT|6zFQNraY^{lI>5&eN6B`GmOZo`hOCu#=zP+5E?gSQsPU38(G{UJVd*O|VL zkOpbO4RyI>g9;~A3#3dSdf%+6#(rpX|60}gPe@oYUN!O~=2t=E;2e`aiLyM21 zjd9txT&~M*WR2orrrved2LrM6j4CJNfwrdk0fUoxmoJPO6z8>#$Ta*{`r|qvtqAw% zU=+&O$&ZbGqnvHJzug^I$(LT=g#4R=HyN$`ZjfMV6WVV9uAYwyah{*dLME>VqL`LL zD(i7CLntYO z*`rGN=|5CCds`-XdGpZrfVYp_pU;hk`3lwvuer{XUlUtReGoy9e@2B-%Zr+;Tn0J# zM@@Me87CD0f}{_fIez$``*mIHBmUV@nAmZhkVdHsq+vj#9))cOATmAjW<6=0vH^)i z&SJKkeE@^0QovC9;*tc0k4(EQt7pzzYXNn#$=NlJ+#kH$oL%dMg5Ts zefOupIa7ESE~$Q8#jDmH%Spmr5sT3O#nDHtsHu_Sxz=#2AuX50Km*`$t_v~I+W=_4 zwzm3tr9+LK>Thjy zM?CC}NSxZBdX-PoSlOo+6*f#evk2wG+jS(+E@l|X?MxSS}%9;`^ftdEBmqI z_&Ha*gMday|MSn<&F)AEyOz8?E;c1I;VQ6PqhH!WA`gq{g4oIR?0%!)usAst>G*Q7 z6Km`U%gX9FUma|1Hb7I=^$}(fQcZE6T%N#l-pNh>VRi5k1ZSSH*)E}zwaO>2jb^xe zxeV_iPioh~l&Ow4oqXvvyC@gp1=fh@qCH>~Z|_Nz@<9vC-!9;@54QiYqGbs(QI^ zVYrnXI>z0iKQq!DLqU@l%)VDat@eZ^=LdMNe=T!jB$!0 zdOk_X^ZS#pwWk(;faYh^LSOQQ25MUBJg89XNzeesB%0Nhl38de_PaG-7?3r+@RIp! zngMsq0gEqDtI&vx1vm7v9rY*b7}Q3jSwpD~%Kro6qhNesBSTP4{UL=T`xiyhxbk*@ z7sj!Xrvd)CIiuulUp$=rXWSyJqkPfiXT~kkxcQyNbrUcS*mAYP`(KYsY&^_12kj)^ zeWImO^fyy4wgYzVVza!(7B=fn?q%}xAa@TMa}{paDMctb_6|7e)IjWEdusEkY%i@2(De@Q)o!r6RZl@Hvk*aRiO&X!p zzQVvR#mnlTu}A_E@mPqe&8Z+&>*R0>zY$XT?@UBJ%^}nKC!ldVNNY{q7G<7&pfV*w z%n^7Z!9>Vn%Q}*-c2_-{{2DT;pqzTQHz~T{a$yagOrTjijA^9ofV$*p0v#M zRqjHDPo=-d#>R;McRgvZKEl#Tzj!Ukedg;@^sE^ZqAct%$q$T(w_xW^pTC$9w;CVW z`L-g6CHBnSoyM*@jZAUK@B^c>c%$Y%ZW2yx-7lY|D!$nLKRwVdg@iJhGLuX6CyR`U zYu)mPe3n_o4BOE=Rj5}3gAtF8w|J272%#=pid*OL=SOS zwj(|q*&9hPm*`99S*Qs68W{4X2FK$nN*Fi=b2%8{j+Q~amEIxp>M2V7;nP6T|Ktm> z=S=V4fv#Zh{31NEY$YG#w)^0MpPabY?~O1~{S4IQ)rvW$WRqLq;Mr_e@U|92@#+G( zZ`%y^t#P7sPsiRPAkjy?renR%V@nNr4`r0*I%lUZ?N1&))h>Sd+pDo=0N54o21vZq z_c};$@lc~1DbrLnV=FE@oh`3EVU5C%VV z;3)Ln30}j*6~Vb~!TkG;5(Vr=!RBMpRVwRB2)Mwono_!}KGo;!x)c~SK8x_C>I44+ zB7HJee;7vzH674hFM4Gv{;xEIPl|fK1a#$f;pp&8_JIkDhYaB`cKilH6dI= zO*Vl|2|dCVtlL{{)tKh}`4}mXQSf)`Sib)3U}IaSC?|{1M1Q&>{!}f!#yQcA`K)^# z0g;c40hb{ISM0ug-)*1f<=*PC)cNZVM+JzQr4V%G%;kZ+yq#BauhWOJ+k)yWTZDM$ zNl$9=sOw`&6H!&!T!+%u)VA?eA={WEK|tp6-94%4^Lh6no}bv1WvuA$(>0;qw*nA& YXK$vBdDGda{jzN7-!#^#&~l3YANteCtpET3 literal 0 HcmV?d00001 From 8dbfe3dac37dfb4294641df8fa2815250b891245 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 8 Jan 2020 14:48:44 -0600 Subject: [PATCH 160/257] sidecar: allow custom http con pool size fix #1953 (#1969) Signed-off-by: Brett Jones --- CHANGELOG.md | 2 ++ cmd/thanos/sidecar.go | 19 +++++++++++++++++-- docs/components/sidecar.md | 5 +++++ pkg/exthttp/transport.go | 24 ++++++++++++++++++++++++ pkg/store/prometheus.go | 3 ++- pkg/store/prometheus_test.go | 10 ++++------ 6 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 pkg/exthttp/transport.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 26b1334c06..6906d7da41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased +- [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags + ### Fixed - [#1919](https://github.com/thanos-io/thanos/issues/1919) Compactor: Fixed potential data loss when uploading older blocks, or upload taking long time while compactor is diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index f6961e9766..52161f32c0 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -3,6 +3,7 @@ package main import ( "context" "math" + "net/http" "net/url" "sync" "time" @@ -18,6 +19,7 @@ import ( "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/exthttp" thanosmodel "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" @@ -30,6 +32,8 @@ import ( "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/tls" + "github.com/thanos-io/thanos/pkg/tracing" + "gopkg.in/alecthomas/kingpin.v2" ) @@ -45,6 +49,9 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { promReadyTimeout := cmd.Flag("prometheus.ready_timeout", "Maximum time to wait for the Prometheus instance to start up"). Default("10m").Duration() + connectionPoolSize := cmd.Flag("receive.connection-pool-size", "Controls the http MaxIdleConns. Default is 0, which is unlimited").Int() + connectionPoolSizePerHost := cmd.Flag("receive.connection-pool-size-per-host", "Controls the http MaxIdleConnsPerHost").Default("100").Int() + dataDir := cmd.Flag("tsdb.path", "Data directory of TSDB."). Default("./data").String() @@ -95,6 +102,8 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { *ignoreBlockSize, component.Sidecar, *minTime, + *connectionPoolSize, + *connectionPoolSizePerHost, ) } } @@ -120,6 +129,8 @@ func runSidecar( ignoreBlockSize bool, comp component.Component, limitMinTime thanosmodel.TimeOrDurationValue, + connectionPoolSize int, + connectionPoolSizePerHost int, ) error { var m = &promMetadata{ promURL: promURL, @@ -243,8 +254,12 @@ func runSidecar( } { - promStore, err := store.NewPrometheusStore( - logger, nil, promURL, component.Sidecar, m.Labels, m.Timestamps) + t := exthttp.NewTransport() + t.MaxIdleConnsPerHost = connectionPoolSizePerHost + t.MaxIdleConns = connectionPoolSize + c := &http.Client{Transport: tracing.HTTPTripperware(logger, t)} + + promStore, err := store.NewPrometheusStore(logger, c, promURL, component.Sidecar, m.Labels, m.Timestamps) if err != nil { return errors.Wrap(err, "create Prometheus store") } diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 4bdd8ced46..4823f7cbe0 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -126,6 +126,11 @@ Flags: --prometheus.ready_timeout=10m Maximum time to wait for the Prometheus instance to start up + --receive.connection-pool-size=RECEIVE.CONNECTION-POOL-SIZE + Controls the http MaxIdleConns. Default is 0, + which is unlimited + --receive.connection-pool-size-per-host=100 + Controls the http MaxIdleConnsPerHost --tsdb.path="./data" Data directory of TSDB. --reloader.config-file="" Config file watched by the reloader. --reloader.config-envsubst-file="" diff --git a/pkg/exthttp/transport.go b/pkg/exthttp/transport.go new file mode 100644 index 0000000000..92725c9b33 --- /dev/null +++ b/pkg/exthttp/transport.go @@ -0,0 +1,24 @@ +package exthttp + +import ( + "net" + "net/http" + "time" +) + +// NewTransport creates a new http.Transport with default settings. +func NewTransport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } +} diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 4012eceb9c..41fa52e3e6 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -27,6 +27,7 @@ import ( "github.com/prometheus/prometheus/storage/remote" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/exthttp" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/tracing" @@ -79,7 +80,7 @@ func NewPrometheusStore( } if client == nil { client = &http.Client{ - Transport: tracing.HTTPTripperware(logger, http.DefaultTransport), + Transport: tracing.HTTPTripperware(logger, exthttp.NewTransport()), } } p := &PrometheusStore{ diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index 5db3ee75cc..bf6fee5cc7 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" @@ -364,8 +365,7 @@ func TestPrometheusStore_Series_MatchExternalLabel_e2e(t *testing.T) { proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return 0, math.MaxInt64 }, - ) + func() (int64, int64) { return 0, math.MaxInt64 }) testutil.Ok(t, err) srv := newStoreSeriesServer(ctx) @@ -410,8 +410,7 @@ func TestPrometheusStore_Info(t *testing.T) { proxy, err := NewPrometheusStore(nil, nil, nil, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return 123, 456 }, - ) + func() (int64, int64) { return 123, 456 }) testutil.Ok(t, err) resp, err := proxy.Info(ctx, &storepb.InfoRequest{}) @@ -489,8 +488,7 @@ func TestPrometheusStore_Series_SplitSamplesIntoChunksWithMaxSizeOfUint16_e2e(t proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return 0, math.MaxInt64 }, - ) + func() (int64, int64) { return 0, math.MaxInt64 }) testutil.Ok(t, err) // We build chunks only for SAMPLES method. Make sure we ask for SAMPLES only. From 1d71579f04f035cc5457e6c889977baa6a6140c7 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 8 Jan 2020 16:43:44 -0600 Subject: [PATCH 161/257] receive: allow enabling local tsdb compaction (#1967) Signed-off-by: Brett Jones --- CHANGELOG.md | 1 + cmd/thanos/receive.go | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6906d7da41..565858fa6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased - [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags +- [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction ### Fixed diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index f7f412800e..c55d328364 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -70,7 +70,9 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { replicationFactor := cmd.Flag("receive.replication-factor", "How many times to replicate incoming write requests.").Default("1").Uint64() - tsdbBlockDuration := modelDuration(cmd.Flag("tsdb.block-duration", "Duration for local TSDB blocks").Default("2h").Hidden()) + tsdbMinBlockDuration := modelDuration(cmd.Flag("tsdb.min-block-duration", "Min duration for local TSDB blocks").Default("2h").Hidden()) + tsdbMaxBlockDuration := modelDuration(cmd.Flag("tsdb.max-block-duration", "Max duration for local TSDB blocks").Default("2h").Hidden()) + ignoreBlockSize := cmd.Flag("shipper.ignore-unequal-block-size", "If true receive will not require min and max block size flags to be set to the same value. Only use this if you want to keep long retention and compaction enabled, as in the worst case it can result in ~2h data loss for your Thanos bucket storage.").Default("false").Hidden().Bool() walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool() @@ -89,8 +91,8 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { } tsdbOpts := &tsdb.Options{ - MinBlockDuration: *tsdbBlockDuration, - MaxBlockDuration: *tsdbBlockDuration, + MinBlockDuration: *tsdbMinBlockDuration, + MaxBlockDuration: *tsdbMaxBlockDuration, RetentionDuration: *retention, NoLockfile: true, WALCompression: *walCompression, @@ -131,6 +133,7 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { *dataDir, objStoreConfig, tsdbOpts, + *ignoreBlockSize, lset, cw, *local, @@ -165,6 +168,7 @@ func runReceive( dataDir string, objStoreConfig *extflag.PathOrContent, tsdbOpts *tsdb.Options, + ignoreBlockSize bool, lset labels.Labels, cw *receive.ConfigWatcher, endpoint string, @@ -208,6 +212,14 @@ func runReceive( upload = false } + if upload && tsdbOpts.MinBlockDuration != tsdbOpts.MaxBlockDuration { + if !ignoreBlockSize { + return errors.Errorf("found that TSDB Max time is %s and Min time is %s. "+ + "Compaction needs to be disabled (tsdb.min-block-duration = tsdb.max-block-duration)", tsdbOpts.MaxBlockDuration, tsdbOpts.MinBlockDuration) + } + level.Warn(logger).Log("msg", "flag to ignore min/max block duration flags differing is being used. If the upload of a 2h block fails and a tsdb compaction happens that block may be missing from your Thanos bucket storage.") + } + // Start all components while we wait for TSDB to open but only load // initial config and mark ourselves as ready after it completed. From 1a419c26105a72b4a22af48502dc20183733e887 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 9 Jan 2020 18:52:09 +0100 Subject: [PATCH 162/257] *: support TLS and authentication for Thanos Ruler queries (#1939) * *: support TLS and authentication for Thanos Ruler queries Signed-off-by: Simon Pasquier * cmd/thanos: apply Bartek's comments Signed-off-by: Simon Pasquier * Rename FanoutClient to Client Signed-off-by: Simon Pasquier * Address a few more nits Signed-off-by: Simon Pasquier --- CHANGELOG.md | 1 + cmd/thanos/rule.go | 290 +++++++++---------- docs/components/rule.md | 47 ++- docs/service-discovery.md | 11 +- pkg/alert/alert.go | 81 +++++- pkg/alert/alert_test.go | 44 +-- pkg/alert/client.go | 268 ----------------- pkg/alert/config.go | 98 +++++++ pkg/alert/{client_test.go => config_test.go} | 46 ++- pkg/http/http.go | 149 +++++++++- pkg/promclient/promclient.go | 57 +++- pkg/query/config.go | 38 +++ scripts/cfggen/main.go | 11 +- test/e2e/rule_test.go | 156 +++++++--- test/e2e/spinup_test.go | 18 +- 15 files changed, 779 insertions(+), 536 deletions(-) delete mode 100644 pkg/alert/client.go create mode 100644 pkg/alert/config.go rename pkg/alert/{client_test.go => config_test.go} (55%) create mode 100644 pkg/query/config.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 565858fa6f..d915df7e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1881](https://github.com/thanos-io/thanos/pull/1881) Store Gateway: memcached support for index cache. See [documentation](docs/components/store.md/#index-cache) for further information. - [#1904](https://github.com/thanos-io/thanos/pull/1904) Add a skip-chunks option in Store Series API to improve the response time of `/api/v1/series` endpoint. - [#1910](https://github.com/thanos-io/thanos/pull/1910) Query: `/api/v1/labels` now understands `POST` - useful for sending bigger requests +- [#1939](https://github.com/thanos-io/thanos/pull/1939) Ruler: Add TLS and authentication support for query endpoints with the `--query.config` and `--query.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. ### Changed diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 395a16ecc8..65430e7849 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -2,7 +2,6 @@ package main import ( "context" - "fmt" "math/rand" "net/http" "net/url" @@ -23,8 +22,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/common/route" - "github.com/prometheus/prometheus/discovery/file" - "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" @@ -33,14 +30,15 @@ import ( "github.com/thanos-io/thanos/pkg/alert" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" - "github.com/thanos-io/thanos/pkg/discovery/cache" "github.com/thanos-io/thanos/pkg/discovery/dns" "github.com/thanos-io/thanos/pkg/extflag" "github.com/thanos-io/thanos/pkg/extprom" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" + http_util "github.com/thanos-io/thanos/pkg/http" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/promclient" + "github.com/thanos-io/thanos/pkg/query" thanosrule "github.com/thanos-io/thanos/pkg/rule" v1 "github.com/thanos-io/thanos/pkg/rule/api" "github.com/thanos-io/thanos/pkg/runutil" @@ -101,7 +99,9 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { queries := cmd.Flag("query", "Addresses of statically configured query API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect query API servers through respective DNS lookups."). PlaceHolder("").Strings() - fileSDFiles := cmd.Flag("query.sd-files", "Path to file that contain addresses of query peers. The path can be a glob pattern (repeatable)."). + queryConfig := extflag.RegisterPathOrContent(cmd, "query.config", "YAML file that contains query API servers configuration. See format details: https://thanos.io/components/rule.md/#configuration. If defined, it takes precedence over the '--query' and '--query.sd-files' flags.", false) + + fileSDFiles := cmd.Flag("query.sd-files", "Path to file that contains addresses of query API servers. The path can be a glob pattern (repeatable)."). PlaceHolder("").Strings() fileSDInterval := modelDuration(cmd.Flag("query.sd-interval", "Refresh interval to re-read file SD files. (used as a fallback)"). @@ -131,6 +131,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { WALCompression: *walCompression, } + // Parse and check query configuration. lookupQueries := map[string]struct{}{} for _, q := range *queries { if _, ok := lookupQueries[q]; ok { @@ -140,17 +141,24 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { lookupQueries[q] = struct{}{} } - var fileSD *file.Discovery - if len(*fileSDFiles) > 0 { - conf := &file.SDConfig{ - Files: *fileSDFiles, - RefreshInterval: *fileSDInterval, - } - fileSD = file.NewDiscovery(conf, logger) + queryConfigYAML, err := queryConfig.Content() + if err != nil { + return err + } + if len(*fileSDFiles) == 0 && len(*queries) == 0 && len(queryConfigYAML) == 0 { + return errors.New("no --query parameter was given") + } + if (len(*fileSDFiles) != 0 || len(*queries) != 0) && len(queryConfigYAML) != 0 { + return errors.New("--query/--query.sd-files and --query.config* parameters cannot be defined at the same time") } - if fileSD == nil && len(*queries) == 0 { - return errors.Errorf("No --query parameter was given.") + // Parse and check alerting configuration. + alertmgrsConfigYAML, err := alertmgrsConfig.Content() + if err != nil { + return err + } + if len(alertmgrsConfigYAML) != 0 && len(*alertmgrs) != 0 { + return errors.New("--alertmanagers.url and --alertmanagers.config* parameters cannot be defined at the same time") } return runRule(g, @@ -160,7 +168,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { lset, *alertmgrs, *alertmgrsTimeout, - alertmgrsConfig, + alertmgrsConfigYAML, time.Duration(*alertmgrsDNSSDInterval), *grpcBindAddr, time.Duration(*grpcGracePeriod), @@ -181,7 +189,9 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { alertQueryURL, *alertExcludeLabels, *queries, - fileSD, + *fileSDFiles, + time.Duration(*fileSDInterval), + queryConfigYAML, time.Duration(*dnsSDInterval), *dnsSDResolver, comp, @@ -199,7 +209,7 @@ func runRule( lset labels.Labels, alertmgrURLs []string, alertmgrsTimeout time.Duration, - alertmgrsConfig *extflag.PathOrContent, + alertmgrsConfigYAML []byte, alertmgrsDNSSDInterval time.Duration, grpcBindAddr string, grpcGracePeriod time.Duration, @@ -220,7 +230,9 @@ func runRule( alertQueryURL *url.URL, alertExcludeLabels []string, queryAddrs []string, - fileSD *file.Discovery, + querySDFiles []string, + querySDInterval time.Duration, + queryConfigYAML []byte, dnsSDInterval time.Duration, dnsSDResolver string, comp component.Component, @@ -259,10 +271,58 @@ func runRule( reg.MustRegister(rulesLoaded) reg.MustRegister(ruleEvalWarnings) - for _, addr := range queryAddrs { - if addr == "" { - return errors.New("static querier address cannot be empty") + var queryCfg []query.Config + if len(queryConfigYAML) > 0 { + var err error + queryCfg, err = query.LoadConfigs(queryConfigYAML) + if err != nil { + return err + } + } else { + for _, addr := range queryAddrs { + if addr == "" { + return errors.New("static querier address cannot be empty") + } + } + + // Build the query configuration from the legacy query flags. + var fileSDConfigs []http_util.FileSDConfig + if len(querySDFiles) > 0 { + fileSDConfigs = append(fileSDConfigs, http_util.FileSDConfig{ + Files: querySDFiles, + RefreshInterval: model.Duration(querySDInterval), + }) + } + queryCfg = append(queryCfg, + query.Config{ + EndpointsConfig: http_util.EndpointsConfig{ + Scheme: "http", + StaticAddresses: queryAddrs, + FileSDConfigs: fileSDConfigs, + }, + }, + ) + } + + queryProvider := dns.NewProvider( + logger, + extprom.WrapRegistererWithPrefix("thanos_ruler_query_apis_", reg), + dns.ResolverType(dnsSDResolver), + ) + var queryClients []*http_util.Client + for _, cfg := range queryCfg { + c, err := http_util.NewHTTPClient(cfg.HTTPClientConfig, "query") + if err != nil { + return err } + c.Transport = tracing.HTTPTripperware(logger, c.Transport) + queryClient, err := http_util.NewClient(logger, cfg.EndpointsConfig, c, queryProvider.Clone()) + if err != nil { + return err + } + queryClients = append(queryClients, queryClient) + // Discover and resolve query addresses. + addDiscoveryGroups(g, queryClient, dnsSDInterval) } db, err := tsdb.Open(dataDir, log.With(logger, "component", "tsdb"), reg, tsdbOpts) @@ -279,28 +339,9 @@ func runRule( }) } - // FileSD query addresses. - fileSDCache := cache.New() - - dnsProvider := dns.NewProvider( - logger, - extprom.WrapRegistererWithPrefix("thanos_ruler_query_apis_", reg), - dns.ResolverType(dnsSDResolver), - ) - // Build the Alertmanager clients. - alertmgrsConfigYAML, err := alertmgrsConfig.Content() - if err != nil { - return err - } - var ( - alertingCfg alert.AlertingConfig - alertmgrs []*alert.Alertmanager - ) + var alertingCfg alert.AlertingConfig if len(alertmgrsConfigYAML) > 0 { - if len(alertmgrURLs) != 0 { - return errors.New("--alertmanagers.url and --alertmanagers.config* flags cannot be defined at the same time") - } alertingCfg, err = alert.LoadAlertingConfig(alertmgrsConfigYAML) if err != nil { return err @@ -308,7 +349,7 @@ func runRule( } else { // Build the Alertmanager configuration from the legacy flags. for _, addr := range alertmgrURLs { - cfg, err := alert.BuildAlertmanagerConfig(logger, addr, alertmgrsTimeout) + cfg, err := alert.BuildAlertmanagerConfig(addr, alertmgrsTimeout) if err != nil { return err } @@ -325,13 +366,22 @@ func runRule( extprom.WrapRegistererWithPrefix("thanos_ruler_alertmanagers_", reg), dns.ResolverType(dnsSDResolver), ) + var alertmgrs []*alert.Alertmanager for _, cfg := range alertingCfg.Alertmanagers { + c, err := http_util.NewHTTPClient(cfg.HTTPClientConfig, "alertmanager") + if err != nil { + return err + } + c.Transport = tracing.HTTPTripperware(logger, c.Transport) // Each Alertmanager client has a different list of targets thus each needs its own DNS provider. - am, err := alert.NewAlertmanager(logger, cfg, amProvider.Clone()) + amClient, err := http_util.NewClient(logger, cfg.EndpointsConfig, c, amProvider.Clone()) if err != nil { return err } - alertmgrs = append(alertmgrs, am) + // Discover and resolve Alertmanager addresses. + addDiscoveryGroups(g, amClient, alertmgrsDNSSDInterval) + + alertmgrs = append(alertmgrs, alert.NewAlertmanager(logger, amClient, time.Duration(cfg.Timeout))) } // Run rule evaluation and alert notifications. @@ -383,7 +433,7 @@ func runRule( opts := opts opts.Registerer = extprom.WrapRegistererWith(prometheus.Labels{"strategy": strings.ToLower(s.String())}, reg) opts.Context = ctx - opts.QueryFunc = queryFunc(logger, dnsProvider, duplicatedQuery, ruleEvalWarnings, s) + opts.QueryFunc = queryFunc(logger, queryClients, duplicatedQuery, ruleEvalWarnings, s) mgr := rules.NewManager(&opts) ruleMgr.SetRuleManager(s, mgr) @@ -398,35 +448,9 @@ func runRule( }) } } - // Discover and resolve Alertmanager addresses. - { - for i := range alertmgrs { - am := alertmgrs[i] - ctx, cancel := context.WithCancel(context.Background()) - g.Add(func() error { - am.Discover(ctx) - return nil - }, func(error) { - cancel() - }) - - g.Add(func() error { - return runutil.Repeat(alertmgrsDNSSDInterval, ctx.Done(), func() error { - am.Resolve(ctx) - return nil - }) - }, func(error) { - cancel() - }) - } - } // Run the alert sender. { - clients := make([]alert.AlertmanagerClient, len(alertmgrs)) - for i := range alertmgrs { - clients[i] = alertmgrs[i] - } - sdr := alert.NewSender(logger, reg, clients) + sdr := alert.NewSender(logger, reg, alertmgrs) ctx, cancel := context.WithCancel(context.Background()) g.Add(func() error { @@ -443,39 +467,6 @@ func runRule( cancel() }) } - // Run File Service Discovery and update the query addresses when the files are modified. - if fileSD != nil { - var fileSDUpdates chan []*targetgroup.Group - ctxRun, cancelRun := context.WithCancel(context.Background()) - - fileSDUpdates = make(chan []*targetgroup.Group) - - g.Add(func() error { - fileSD.Run(ctxRun, fileSDUpdates) - return nil - }, func(error) { - cancelRun() - }) - - ctxUpdate, cancelUpdate := context.WithCancel(context.Background()) - g.Add(func() error { - for { - select { - case update := <-fileSDUpdates: - // Discoverers sometimes send nil updates so need to check for it to avoid panics. - if update == nil { - continue - } - fileSDCache.Update(update) - case <-ctxUpdate.Done(): - return nil - } - } - }, func(error) { - cancelUpdate() - close(fileSDUpdates) - }) - } // Handle reload and termination interrupts. reload := make(chan struct{}, 1) @@ -546,18 +537,6 @@ func runRule( close(cancel) }) } - // Periodically update the addresses from static flags and file SD by resolving them using DNS SD if necessary. - { - ctx, cancel := context.WithCancel(context.Background()) - g.Add(func() error { - return runutil.Repeat(dnsSDInterval, ctx.Done(), func() error { - dnsProvider.Resolve(ctx, append(fileSDCache.Addresses(), queryAddrs...)) - return nil - }) - }, func(error) { - cancel() - }) - } statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Start gRPC server. { @@ -702,17 +681,17 @@ func labelsTSDBToProm(lset labels.Labels) (res labels.Labels) { return res } -func removeDuplicateQueryAddrs(logger log.Logger, duplicatedQueriers prometheus.Counter, addrs []string) []string { +func removeDuplicateQueryEndpoints(logger log.Logger, duplicatedQueriers prometheus.Counter, urls []*url.URL) []*url.URL { set := make(map[string]struct{}) - deduplicated := make([]string, 0, len(addrs)) - for _, addr := range addrs { - if _, ok := set[addr]; ok { - level.Warn(logger).Log("msg", "duplicate query address is provided - %v", addr) + deduplicated := make([]*url.URL, 0, len(urls)) + for _, u := range urls { + if _, ok := set[u.String()]; ok { + level.Warn(logger).Log("msg", "duplicate query address is provided - %v", u.String()) duplicatedQueriers.Inc() continue } - deduplicated = append(deduplicated, addr) - set[addr] = struct{}{} + deduplicated = append(deduplicated, u) + set[u.String()] = struct{}{} } return deduplicated } @@ -721,7 +700,7 @@ func removeDuplicateQueryAddrs(logger log.Logger, duplicatedQueriers prometheus. // back or the context get canceled. func queryFunc( logger log.Logger, - dnsProvider *dns.Provider, + queriers []*http_util.Client, duplicatedQuery prometheus.Counter, ruleEvalWarnings *prometheus.CounterVec, partialResponseStrategy storepb.PartialResponseStrategy, @@ -738,29 +717,27 @@ func queryFunc( panic(errors.Errorf("unknown partial response strategy %v", partialResponseStrategy).Error()) } - return func(ctx context.Context, q string, t time.Time) (promql.Vector, error) { - // Add DNS resolved addresses from static flags and file SD. - // TODO(bwplotka): Consider generating addresses in *url.URL. - addrs := dnsProvider.Addresses() - - addrs = removeDuplicateQueryAddrs(logger, duplicatedQuery, addrs) - - for _, i := range rand.Perm(len(addrs)) { - u, err := url.Parse(fmt.Sprintf("http://%s", addrs[i])) - if err != nil { - return nil, errors.Wrapf(err, "url parse %s", addrs[i]) - } + promClients := make([]*promclient.Client, 0, len(queriers)) + for _, q := range queriers { + promClients = append(promClients, promclient.NewClient(logger, q)) + } - span, ctx := tracing.StartSpan(ctx, spanID) - v, warns, err := promclient.PromqlQueryInstant(ctx, logger, u, q, t, promclient.QueryOptions{ - Deduplicate: true, - PartialResponseStrategy: partialResponseStrategy, - }) - span.Finish() + return func(ctx context.Context, q string, t time.Time) (promql.Vector, error) { + for _, i := range rand.Perm(len(queriers)) { + promClient := promClients[i] + endpoints := removeDuplicateQueryEndpoints(logger, duplicatedQuery, queriers[i].Endpoints()) + for _, i := range rand.Perm(len(endpoints)) { + span, ctx := tracing.StartSpan(ctx, spanID) + v, warns, err := promClient.PromqlQueryInstant(ctx, endpoints[i], q, t, promclient.QueryOptions{ + Deduplicate: true, + PartialResponseStrategy: partialResponseStrategy, + }) + span.Finish() - if err != nil { - level.Error(logger).Log("err", err, "query", q) - } else { + if err != nil { + level.Error(logger).Log("err", err, "query", q) + continue + } if len(warns) > 0 { ruleEvalWarnings.WithLabelValues(strings.ToLower(partialResponseStrategy.String())).Inc() // TODO(bwplotka): Propagate those to UI, probably requires changing rule manager code ): @@ -769,6 +746,25 @@ func queryFunc( return v, nil } } - return nil, errors.Errorf("no query peer reachable") + return nil, errors.Errorf("no query API server reachable") } } + +func addDiscoveryGroups(g *run.Group, c *http_util.Client, interval time.Duration) { + ctx, cancel := context.WithCancel(context.Background()) + g.Add(func() error { + c.Discover(ctx) + return nil + }, func(error) { + cancel() + }) + + g.Add(func() error { + return runutil.Repeat(interval, ctx.Done(), func() error { + c.Resolve(ctx) + return nil + }) + }, func(error) { + cancel() + }) +} diff --git a/docs/components/rule.md b/docs/components/rule.md index 093c17726c..739bc60b50 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -360,9 +360,22 @@ Flags: prefixed with 'dns+' or 'dnssrv+' to detect query API servers through respective DNS lookups. + --query.config-file= + Path to YAML file that contains query API + servers configuration. See format details: + https://thanos.io/components/rule.md/#configuration. + If defined, it takes precedence over the + '--query' and '--query.sd-files' flags. + --query.config= Alternative to 'query.config-file' flag (lower + priority). Content of YAML file that contains + query API servers configuration. See format + details: + https://thanos.io/components/rule.md/#configuration. + If defined, it takes precedence over the + '--query' and '--query.sd-files' flags. --query.sd-files= ... - Path to file that contain addresses of query - peers. The path can be a glob pattern + Path to file that contains addresses of query + API servers. The path can be a glob pattern (repeatable). --query.sd-interval=5m Refresh interval to re-read file SD files. (used as a fallback) @@ -404,3 +417,33 @@ alertmanagers: path_prefix: "" timeout: 10s ``` + +### Query API + +The `--query.config` and `--query.config-file` flags allow specifying multiple query endpoints. Those entries are treated as a single HA group. This means that query failure is claimed only if the Ruler fails to query all instances. + +The configuration format is the following: + +[embedmd]:# (../flags/config_rule_query.txt yaml) +```yaml +- http_config: + basic_auth: + username: "" + password: "" + password_file: "" + bearer_token: "" + bearer_token_file: "" + proxy_url: "" + tls_config: + ca_file: "" + cert_file: "" + key_file: "" + server_name: "" + insecure_skip_verify: false + static_configs: [] + file_sd_configs: + - files: [] + refresh_interval: 0s + scheme: http + path_prefix: "" +``` diff --git a/docs/service-discovery.md b/docs/service-discovery.md index 34cf0914cd..2fa366dd6a 100644 --- a/docs/service-discovery.md +++ b/docs/service-discovery.md @@ -31,9 +31,9 @@ The repeatable flag `--store=` can be used to specify a `StoreAPI` that ` ### Thanos Rule -The repeatable flag `--query=` can be used to specify a `QueryAPI` that `Thanos Rule` should use. +`Thanos Rule` supports the configuration of `QueryAPI` endpoints using YAML with the `--query.config=` and `--query.config-file=` flags in the `static_configs` section. -`Thanos Rule` also supports the configuration of Alertmanager endpoints using YAML with the `--alertmanagers.config=` and `--alertmanagers.config-file=` flags in the `StaticAddress` section. +`Thanos Rule` also supports the configuration of Alertmanager endpoints using YAML with the `--alertmanagers.config=` and `--alertmanagers.config-file=` flags in the `static_configs` section. ## File Service Discovery @@ -72,12 +72,9 @@ The flag `--store.sd-interval=<5m>` can be used to change the fallback re-read i ### Thanos Rule -The repeatable flag `--query.sd-files=` can be used to specify the path to files that contain addresses of `QueryAPI` servers. -Again, the `` can be a glob pattern. +`Thanos Rule` supports the configuration of `QueryAPI` endpoints using YAML with the `--query.config=` and `--query.config-file=` flags in the `file_sd_configs` section. -The flag `--query.sd-interval=<5m>` can be used to change the fallback re-read interval. - -`Thanos Rule` also supports the configuration of Alertmanager endpoints using YAML with the `--alertmanagers.config=` and `--alertmanagers.config-file=` flags in the `FileSDfiles` section.. +`Thanos Rule` also supports the configuration of Alertmanager endpoints using YAML with the `--alertmanagers.config=` and `--alertmanagers.config-file=` flags in the `file_sd_configs` section. ## DNS Service Discovery diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go index 2a9e2e8ab8..2387a2245c 100644 --- a/pkg/alert/alert.go +++ b/pkg/alert/alert.go @@ -7,16 +7,26 @@ import ( "encoding/json" "fmt" "io" + "net/http" "net/url" + "path" "sync" "sync/atomic" "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/prometheus/pkg/labels" + + "github.com/thanos-io/thanos/pkg/runutil" +) + +const ( + defaultAlertmanagerPort = 9093 + alertPushEndpoint = "/api/v1/alerts" + contentTypeJSON = "application/json" ) // Alert is a generic representation of an alert in the Prometheus eco-system. @@ -240,15 +250,10 @@ func (q *Queue) Push(alerts []*Alert) { } } -type AlertmanagerClient interface { - Endpoints() []*url.URL - Do(context.Context, *url.URL, io.Reader) error -} - // Sender sends notifications to a dynamic set of alertmanagers. type Sender struct { logger log.Logger - alertmanagers []AlertmanagerClient + alertmanagers []*Alertmanager sent *prometheus.CounterVec errs *prometheus.CounterVec @@ -261,7 +266,7 @@ type Sender struct { func NewSender( logger log.Logger, reg prometheus.Registerer, - alertmanagers []AlertmanagerClient, + alertmanagers []*Alertmanager, ) *Sender { if logger == nil { logger = log.NewNopLogger() @@ -312,15 +317,15 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { wg sync.WaitGroup numSuccess uint64 ) - for _, amc := range s.alertmanagers { - for _, u := range amc.Endpoints() { + for _, am := range s.alertmanagers { + for _, u := range am.dispatcher.Endpoints() { wg.Add(1) - go func(amc AlertmanagerClient, u *url.URL) { + go func(am *Alertmanager, u *url.URL) { defer wg.Done() level.Debug(s.logger).Log("msg", "sending alerts", "alertmanager", u.Host, "numAlerts", len(alerts)) start := time.Now() - if err := amc.Do(ctx, u, bytes.NewReader(b)); err != nil { + if err := am.postAlerts(ctx, *u, bytes.NewReader(b)); err != nil { level.Warn(s.logger).Log( "msg", "sending alerts failed", "alertmanager", u.Host, @@ -334,7 +339,7 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { s.sent.WithLabelValues(u.Host).Add(float64(len(alerts))) atomic.AddUint64(&numSuccess, 1) - }(amc, u) + }(am, u) } } wg.Wait() @@ -346,3 +351,53 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { s.dropped.Add(float64(len(alerts))) level.Warn(s.logger).Log("msg", "failed to send alerts to all alertmanagers", "alerts", string(b)) } + +type Dispatcher interface { + // Endpoints returns the list of endpoint URLs the dispatcher knows about. + Endpoints() []*url.URL + // Do sends an HTTP request and returns a response. + Do(*http.Request) (*http.Response, error) +} + +// Alertmanager is an HTTP client that can send alerts to a cluster of Alertmanager endpoints. +type Alertmanager struct { + logger log.Logger + dispatcher Dispatcher + timeout time.Duration +} + +// NewAlertmanager returns a new Alertmanager client. +func NewAlertmanager(logger log.Logger, dispatcher Dispatcher, timeout time.Duration) *Alertmanager { + if logger == nil { + logger = log.NewNopLogger() + } + + return &Alertmanager{ + logger: logger, + dispatcher: dispatcher, + timeout: timeout, + } +} + +func (a *Alertmanager) postAlerts(ctx context.Context, u url.URL, r io.Reader) error { + u.Path = path.Join(u.Path, alertPushEndpoint) + req, err := http.NewRequest("POST", u.String(), r) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, a.timeout) + defer cancel() + req = req.WithContext(ctx) + req.Header.Set("Content-Type", contentTypeJSON) + + resp, err := a.dispatcher.Do(req) + if err != nil { + return errors.Wrapf(err, "send request to %q", u.String()) + } + defer runutil.ExhaustCloseWithLogOnErr(a.logger, resp.Body, "send one alert") + + if resp.StatusCode/100 != 2 { + return errors.Errorf("bad response status %v from %q", resp.Status, u.String()) + } + return nil +} diff --git a/pkg/alert/alert_test.go b/pkg/alert/alert_test.go index 7e1e851d80..c3fa5f7cb6 100644 --- a/pkg/alert/alert_test.go +++ b/pkg/alert/alert_test.go @@ -2,14 +2,16 @@ package alert import ( "context" - "io" + "net/http" + "net/http/httptest" "net/url" "sync" "testing" + "time" + "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/labels" - "github.com/pkg/errors" promtestutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -48,31 +50,34 @@ func assertSameHosts(t *testing.T, expected []*url.URL, found []*url.URL) { } type fakeClient struct { - urls []*url.URL - postf func(u *url.URL) error - mtx sync.Mutex - seen []*url.URL + urls []*url.URL + dof func(u *url.URL) (*http.Response, error) + mtx sync.Mutex + seen []*url.URL } func (f *fakeClient) Endpoints() []*url.URL { return f.urls } -func (f *fakeClient) Do(ctx context.Context, u *url.URL, r io.Reader) error { +func (f *fakeClient) Do(req *http.Request) (*http.Response, error) { f.mtx.Lock() defer f.mtx.Unlock() + u := req.URL f.seen = append(f.seen, u) - if f.postf == nil { - return nil + if f.dof == nil { + rec := httptest.NewRecorder() + rec.WriteHeader(http.StatusOK) + return rec.Result(), nil } - return f.postf(u) + return f.dof(u) } func TestSenderSendsOk(t *testing.T) { poster := &fakeClient{ urls: []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}}, } - s := NewSender(nil, nil, []AlertmanagerClient{poster}) + s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute)}) s.Send(context.Background(), []*Alert{{}, {}}) @@ -89,14 +94,17 @@ func TestSenderSendsOk(t *testing.T) { func TestSenderSendsOneFails(t *testing.T) { poster := &fakeClient{ urls: []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}}, - postf: func(u *url.URL) error { + dof: func(u *url.URL) (*http.Response, error) { + rec := httptest.NewRecorder() if u.Host == "am1:9090" { - return errors.New("no such host") + rec.WriteHeader(http.StatusBadRequest) + } else { + rec.WriteHeader(http.StatusOK) } - return nil + return rec.Result(), nil }, } - s := NewSender(nil, nil, []AlertmanagerClient{poster}) + s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute)}) s.Send(context.Background(), []*Alert{{}, {}}) @@ -113,11 +121,11 @@ func TestSenderSendsOneFails(t *testing.T) { func TestSenderSendsAllFail(t *testing.T) { poster := &fakeClient{ urls: []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}}, - postf: func(u *url.URL) error { - return errors.New("no such host") + dof: func(u *url.URL) (*http.Response, error) { + return nil, errors.New("no such host") }, } - s := NewSender(nil, nil, []AlertmanagerClient{poster}) + s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute)}) s.Send(context.Background(), []*Alert{{}, {}}) diff --git a/pkg/alert/client.go b/pkg/alert/client.go deleted file mode 100644 index 679f0f7f2e..0000000000 --- a/pkg/alert/client.go +++ /dev/null @@ -1,268 +0,0 @@ -package alert - -import ( - "context" - "io" - "net" - "net/http" - "net/url" - "path" - "strconv" - "strings" - "sync" - "time" - - "github.com/go-kit/kit/log" - "github.com/pkg/errors" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/discovery/file" - "github.com/prometheus/prometheus/discovery/targetgroup" - "gopkg.in/yaml.v2" - - "github.com/thanos-io/thanos/pkg/discovery/cache" - "github.com/thanos-io/thanos/pkg/discovery/dns" - http_util "github.com/thanos-io/thanos/pkg/http" - "github.com/thanos-io/thanos/pkg/runutil" -) - -const ( - defaultAlertmanagerPort = 9093 - alertPushEndpoint = "/api/v1/alerts" - contentTypeJSON = "application/json" -) - -type AlertingConfig struct { - Alertmanagers []AlertmanagerConfig `yaml:"alertmanagers"` -} - -// AlertmanagerConfig represents a client to a cluster of Alertmanager endpoints. -// TODO(simonpasquier): add support for API version (v1 or v2). -type AlertmanagerConfig struct { - // HTTP client configuration. - HTTPClientConfig http_util.ClientConfig `yaml:"http_config"` - - // List of addresses with DNS prefixes. - StaticAddresses []string `yaml:"static_configs"` - // List of file configurations (our FileSD supports different DNS lookups). - FileSDConfigs []FileSDConfig `yaml:"file_sd_configs"` - - // The URL scheme to use when talking to Alertmanagers. - Scheme string `yaml:"scheme"` - - // Path prefix to add in front of the push endpoint path. - PathPrefix string `yaml:"path_prefix"` - - // The timeout used when sending alerts (default: 10s). - Timeout model.Duration `yaml:"timeout"` -} - -type FileSDConfig struct { - Files []string `yaml:"files"` - RefreshInterval model.Duration `yaml:"refresh_interval"` -} - -func (c FileSDConfig) convert() (file.SDConfig, error) { - var fileSDConfig file.SDConfig - b, err := yaml.Marshal(c) - if err != nil { - return fileSDConfig, err - } - err = yaml.Unmarshal(b, &fileSDConfig) - if err != nil { - return fileSDConfig, err - } - return fileSDConfig, nil -} - -func DefaultAlertmanagerConfig() AlertmanagerConfig { - return AlertmanagerConfig{ - Scheme: "http", - Timeout: model.Duration(time.Second * 10), - StaticAddresses: []string{}, - FileSDConfigs: []FileSDConfig{}, - } -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultAlertmanagerConfig() - type plain AlertmanagerConfig - return unmarshal((*plain)(c)) -} - -type AddressProvider interface { - Resolve(context.Context, []string) - Addresses() []string -} - -// Alertmanager represents an HTTP client that can send alerts to a cluster of Alertmanager endpoints. -type Alertmanager struct { - logger log.Logger - - client *http.Client - timeout time.Duration - scheme string - prefix string - - staticAddresses []string - fileSDCache *cache.Cache - fileDiscoverers []*file.Discovery - - provider AddressProvider -} - -// NewAlertmanager returns a new Alertmanager client. -func NewAlertmanager(logger log.Logger, cfg AlertmanagerConfig, provider AddressProvider) (*Alertmanager, error) { - if logger == nil { - logger = log.NewNopLogger() - } - - client, err := http_util.NewClient(cfg.HTTPClientConfig, "alertmanager") - if err != nil { - return nil, err - } - - var discoverers []*file.Discovery - for _, sdCfg := range cfg.FileSDConfigs { - fileSDCfg, err := sdCfg.convert() - if err != nil { - return nil, err - } - discoverers = append(discoverers, file.NewDiscovery(&fileSDCfg, logger)) - } - return &Alertmanager{ - logger: logger, - client: client, - scheme: cfg.Scheme, - prefix: cfg.PathPrefix, - timeout: time.Duration(cfg.Timeout), - staticAddresses: cfg.StaticAddresses, - fileSDCache: cache.New(), - fileDiscoverers: discoverers, - provider: provider, - }, nil -} - -// LoadAlertmanagerConfigs loads a list of AlertmanagerConfig from YAML data. -func LoadAlertingConfig(confYaml []byte) (AlertingConfig, error) { - var cfg AlertingConfig - if err := yaml.UnmarshalStrict(confYaml, &cfg); err != nil { - return cfg, err - } - return cfg, nil -} - -// BuildAlertmanagerConfig initializes and returns an Alertmanager client configuration from a static address. -func BuildAlertmanagerConfig(logger log.Logger, address string, timeout time.Duration) (AlertmanagerConfig, error) { - parsed, err := url.Parse(address) - if err != nil { - return AlertmanagerConfig{}, err - } - - scheme := parsed.Scheme - host := parsed.Host - for _, qType := range []dns.QType{dns.A, dns.SRV, dns.SRVNoA} { - prefix := string(qType) + "+" - if strings.HasPrefix(strings.ToLower(scheme), prefix) { - // Scheme is of the form "+". - scheme = strings.TrimPrefix(scheme, prefix) - host = prefix + parsed.Host - if qType == dns.A { - if _, _, err := net.SplitHostPort(parsed.Host); err != nil { - // The host port could be missing. Append the defaultAlertmanagerPort. - host = host + ":" + strconv.Itoa(defaultAlertmanagerPort) - } - } - break - } - } - var basicAuth http_util.BasicAuth - if parsed.User != nil && parsed.User.String() != "" { - basicAuth.Username = parsed.User.Username() - pw, _ := parsed.User.Password() - basicAuth.Password = pw - } - - return AlertmanagerConfig{ - PathPrefix: parsed.Path, - Scheme: scheme, - StaticAddresses: []string{host}, - Timeout: model.Duration(timeout), - HTTPClientConfig: http_util.ClientConfig{ - BasicAuth: basicAuth, - }, - }, nil -} - -// Endpoints returns the list of known Alertmanager endpoints. -func (a *Alertmanager) Endpoints() []*url.URL { - var urls []*url.URL - for _, addr := range a.provider.Addresses() { - urls = append(urls, - &url.URL{ - Scheme: a.scheme, - Host: addr, - Path: path.Join("/", a.prefix, alertPushEndpoint), - }, - ) - } - return urls -} - -// Do sends a POST request to the given URL. -func (a *Alertmanager) Do(ctx context.Context, u *url.URL, r io.Reader) error { - req, err := http.NewRequest("POST", u.String(), r) - if err != nil { - return err - } - ctx, cancel := context.WithTimeout(ctx, a.timeout) - defer cancel() - req = req.WithContext(ctx) - req.Header.Set("Content-Type", contentTypeJSON) - - resp, err := a.client.Do(req) - if err != nil { - return errors.Wrapf(err, "send request to %q", u) - } - defer runutil.ExhaustCloseWithLogOnErr(a.logger, resp.Body, "send one alert") - - if resp.StatusCode/100 != 2 { - return errors.Errorf("bad response status %v from %q", resp.Status, u) - } - return nil -} - -// Discover runs the service to discover target endpoints. -func (a *Alertmanager) Discover(ctx context.Context) { - var wg sync.WaitGroup - ch := make(chan []*targetgroup.Group) - - for _, d := range a.fileDiscoverers { - wg.Add(1) - go func(d *file.Discovery) { - d.Run(ctx, ch) - wg.Done() - }(d) - } - - func() { - for { - select { - case update := <-ch: - // Discoverers sometimes send nil updates so need to check for it to avoid panics. - if update == nil { - continue - } - a.fileSDCache.Update(update) - case <-ctx.Done(): - return - } - } - }() - wg.Wait() -} - -// Resolve refreshes and resolves the list of Alertmanager targets. -func (a *Alertmanager) Resolve(ctx context.Context) { - a.provider.Resolve(ctx, append(a.fileSDCache.Addresses(), a.staticAddresses...)) -} diff --git a/pkg/alert/config.go b/pkg/alert/config.go new file mode 100644 index 0000000000..3df4216f5c --- /dev/null +++ b/pkg/alert/config.go @@ -0,0 +1,98 @@ +package alert + +import ( + "net" + "net/url" + "strconv" + "strings" + "time" + + "github.com/prometheus/common/model" + "gopkg.in/yaml.v2" + + "github.com/thanos-io/thanos/pkg/discovery/dns" + http_util "github.com/thanos-io/thanos/pkg/http" +) + +type AlertingConfig struct { + Alertmanagers []AlertmanagerConfig `yaml:"alertmanagers"` +} + +// AlertmanagerConfig represents a client to a cluster of Alertmanager endpoints. +// TODO(simonpasquier): add support for API version (v1 or v2). +type AlertmanagerConfig struct { + HTTPClientConfig http_util.ClientConfig `yaml:"http_config"` + EndpointsConfig http_util.EndpointsConfig `yaml:",inline"` + Timeout model.Duration `yaml:"timeout"` +} + +func DefaultAlertmanagerConfig() AlertmanagerConfig { + return AlertmanagerConfig{ + EndpointsConfig: http_util.EndpointsConfig{ + Scheme: "http", + StaticAddresses: []string{}, + FileSDConfigs: []http_util.FileSDConfig{}, + }, + Timeout: model.Duration(time.Second * 10), + } +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultAlertmanagerConfig() + type plain AlertmanagerConfig + return unmarshal((*plain)(c)) +} + +// LoadAlertingConfig loads a list of AlertmanagerConfig from YAML data. +func LoadAlertingConfig(confYaml []byte) (AlertingConfig, error) { + var cfg AlertingConfig + if err := yaml.UnmarshalStrict(confYaml, &cfg); err != nil { + return cfg, err + } + return cfg, nil +} + +// BuildAlertmanagerConfig initializes and returns an Alertmanager client configuration from a static address. +func BuildAlertmanagerConfig(address string, timeout time.Duration) (AlertmanagerConfig, error) { + parsed, err := url.Parse(address) + if err != nil { + return AlertmanagerConfig{}, err + } + + scheme := parsed.Scheme + host := parsed.Host + for _, qType := range []dns.QType{dns.A, dns.SRV, dns.SRVNoA} { + prefix := string(qType) + "+" + if strings.HasPrefix(strings.ToLower(scheme), prefix) { + // Scheme is of the form "+". + scheme = strings.TrimPrefix(scheme, prefix) + host = prefix + parsed.Host + if qType == dns.A { + if _, _, err := net.SplitHostPort(parsed.Host); err != nil { + // The host port could be missing. Append the defaultAlertmanagerPort. + host = host + ":" + strconv.Itoa(defaultAlertmanagerPort) + } + } + break + } + } + var basicAuth http_util.BasicAuth + if parsed.User != nil && parsed.User.String() != "" { + basicAuth.Username = parsed.User.Username() + pw, _ := parsed.User.Password() + basicAuth.Password = pw + } + + return AlertmanagerConfig{ + HTTPClientConfig: http_util.ClientConfig{ + BasicAuth: basicAuth, + }, + EndpointsConfig: http_util.EndpointsConfig{ + PathPrefix: parsed.Path, + Scheme: scheme, + StaticAddresses: []string{host}, + }, + Timeout: model.Duration(timeout), + }, nil +} diff --git a/pkg/alert/client_test.go b/pkg/alert/config_test.go similarity index 55% rename from pkg/alert/client_test.go rename to pkg/alert/config_test.go index d83ca7c275..0a50d4da36 100644 --- a/pkg/alert/client_test.go +++ b/pkg/alert/config_test.go @@ -18,44 +18,56 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { { address: "http://localhost:9093", expected: AlertmanagerConfig{ - StaticAddresses: []string{"localhost:9093"}, - Scheme: "http", + EndpointsConfig: http.EndpointsConfig{ + StaticAddresses: []string{"localhost:9093"}, + Scheme: "http", + }, }, }, { address: "https://am.example.com", expected: AlertmanagerConfig{ - StaticAddresses: []string{"am.example.com"}, - Scheme: "https", + EndpointsConfig: http.EndpointsConfig{ + StaticAddresses: []string{"am.example.com"}, + Scheme: "https", + }, }, }, { address: "dns+http://localhost:9093", expected: AlertmanagerConfig{ - StaticAddresses: []string{"dns+localhost:9093"}, - Scheme: "http", + EndpointsConfig: http.EndpointsConfig{ + StaticAddresses: []string{"dns+localhost:9093"}, + Scheme: "http", + }, }, }, { address: "dnssrv+http://localhost", expected: AlertmanagerConfig{ - StaticAddresses: []string{"dnssrv+localhost"}, - Scheme: "http", + EndpointsConfig: http.EndpointsConfig{ + StaticAddresses: []string{"dnssrv+localhost"}, + Scheme: "http", + }, }, }, { address: "ssh+http://localhost", expected: AlertmanagerConfig{ - StaticAddresses: []string{"localhost"}, - Scheme: "ssh+http", + EndpointsConfig: http.EndpointsConfig{ + StaticAddresses: []string{"localhost"}, + Scheme: "ssh+http", + }, }, }, { address: "dns+https://localhost/path/prefix/", expected: AlertmanagerConfig{ - StaticAddresses: []string{"dns+localhost:9093"}, - Scheme: "https", - PathPrefix: "/path/prefix/", + EndpointsConfig: http.EndpointsConfig{ + StaticAddresses: []string{"dns+localhost:9093"}, + Scheme: "https", + PathPrefix: "/path/prefix/", + }, }, }, { @@ -67,8 +79,10 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { Password: "pass", }, }, - StaticAddresses: []string{"localhost:9093"}, - Scheme: "http", + EndpointsConfig: http.EndpointsConfig{ + StaticAddresses: []string{"localhost:9093"}, + Scheme: "http", + }, }, }, { @@ -77,7 +91,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { }, } { t.Run(tc.address, func(t *testing.T) { - cfg, err := BuildAlertmanagerConfig(nil, tc.address, time.Duration(0)) + cfg, err := BuildAlertmanagerConfig(tc.address, time.Duration(0)) if tc.err { testutil.NotOk(t, err) return diff --git a/pkg/http/http.go b/pkg/http/http.go index 8f096d9b84..8686152139 100644 --- a/pkg/http/http.go +++ b/pkg/http/http.go @@ -2,12 +2,22 @@ package http import ( + "context" "fmt" "net/http" + "net/url" + "path" + "sync" + "github.com/go-kit/kit/log" config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" "github.com/prometheus/common/version" + "github.com/prometheus/prometheus/discovery/file" + "github.com/prometheus/prometheus/discovery/targetgroup" "gopkg.in/yaml.v2" + + "github.com/thanos-io/thanos/pkg/discovery/cache" ) // ClientConfig configures an HTTP client. @@ -50,8 +60,8 @@ func (b BasicAuth) IsZero() bool { return b.Username == "" && b.Password == "" && b.PasswordFile == "" } -// NewClient returns a new HTTP client. -func NewClient(cfg ClientConfig, name string) (*http.Client, error) { +// NewHTTPClient returns a new HTTP client. +func NewHTTPClient(cfg ClientConfig, name string) (*http.Client, error) { httpClientConfig := config_util.HTTPClientConfig{ BearerToken: config_util.Secret(cfg.BearerToken), BearerTokenFile: cfg.BearerTokenFile, @@ -114,3 +124,138 @@ func (u userAgentRoundTripper) RoundTrip(r *http.Request) (*http.Response, error } return u.rt.RoundTrip(r) } + +// EndpointsConfig configures a cluster of HTTP endpoints from static addresses and +// file service discovery. +type EndpointsConfig struct { + // List of addresses with DNS prefixes. + StaticAddresses []string `yaml:"static_configs"` + // List of file configurations (our FileSD supports different DNS lookups). + FileSDConfigs []FileSDConfig `yaml:"file_sd_configs"` + + // The URL scheme to use when talking to targets. + Scheme string `yaml:"scheme"` + + // Path prefix to add in front of the endpoint path. + PathPrefix string `yaml:"path_prefix"` +} + +// FileSDConfig represents a file service discovery configuration. +type FileSDConfig struct { + Files []string `yaml:"files"` + RefreshInterval model.Duration `yaml:"refresh_interval"` +} + +func (c FileSDConfig) convert() (file.SDConfig, error) { + var fileSDConfig file.SDConfig + b, err := yaml.Marshal(c) + if err != nil { + return fileSDConfig, err + } + err = yaml.Unmarshal(b, &fileSDConfig) + if err != nil { + return fileSDConfig, err + } + return fileSDConfig, nil +} + +type AddressProvider interface { + Resolve(context.Context, []string) + Addresses() []string +} + +// Client represents a client that can send requests to a cluster of HTTP-based endpoints. +type Client struct { + logger log.Logger + + httpClient *http.Client + scheme string + prefix string + + staticAddresses []string + fileSDCache *cache.Cache + fileDiscoverers []*file.Discovery + + provider AddressProvider +} + +// NewClient returns a new Client. +func NewClient(logger log.Logger, cfg EndpointsConfig, client *http.Client, provider AddressProvider) (*Client, error) { + if logger == nil { + logger = log.NewNopLogger() + } + + var discoverers []*file.Discovery + for _, sdCfg := range cfg.FileSDConfigs { + fileSDCfg, err := sdCfg.convert() + if err != nil { + return nil, err + } + discoverers = append(discoverers, file.NewDiscovery(&fileSDCfg, logger)) + } + return &Client{ + logger: logger, + httpClient: client, + scheme: cfg.Scheme, + prefix: cfg.PathPrefix, + staticAddresses: cfg.StaticAddresses, + fileSDCache: cache.New(), + fileDiscoverers: discoverers, + provider: provider, + }, nil +} + +// Do executes an HTTP request with the underlying HTTP client. +func (c *Client) Do(req *http.Request) (*http.Response, error) { + return c.httpClient.Do(req) +} + +// Endpoints returns the list of known endpoints. +func (c *Client) Endpoints() []*url.URL { + var urls []*url.URL + for _, addr := range c.provider.Addresses() { + urls = append(urls, + &url.URL{ + Scheme: c.scheme, + Host: addr, + Path: path.Join("/", c.prefix), + }, + ) + } + return urls +} + +// Discover runs the service to discover endpoints until the given context is done. +func (c *Client) Discover(ctx context.Context) { + var wg sync.WaitGroup + ch := make(chan []*targetgroup.Group) + + for _, d := range c.fileDiscoverers { + wg.Add(1) + go func(d *file.Discovery) { + d.Run(ctx, ch) + wg.Done() + }(d) + } + + func() { + for { + select { + case update := <-ch: + // Discoverers sometimes send nil updates so need to check for it to avoid panics. + if update == nil { + continue + } + c.fileSDCache.Update(update) + case <-ctx.Done(): + return + } + } + }() + wg.Wait() +} + +// Resolve refreshes and resolves the list of targets. +func (c *Client) Resolve(ctx context.Context) { + c.provider.Resolve(ctx, append(c.fileSDCache.Addresses(), c.staticAddresses...)) +} diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 8474a161b5..b80e6819d9 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -281,12 +281,42 @@ func (p *QueryOptions) AddTo(values url.Values) error { return nil } -// QueryInstant performs instant query and returns results in model.Vector type. -func QueryInstant(ctx context.Context, logger log.Logger, base *url.URL, query string, t time.Time, opts QueryOptions) (model.Vector, []string, error) { +// HTTPClient sends an HTTP request and returns the response. +type HTTPClient interface { + Do(*http.Request) (*http.Response, error) +} + +// Client represents a Prometheus API client. +type Client struct { + httpClient HTTPClient + logger log.Logger +} + +// NewClient returns a new Prometheus API client. +func NewClient(logger log.Logger, c HTTPClient) *Client { if logger == nil { logger = log.NewNopLogger() } + return &Client{ + httpClient: c, + logger: logger, + } +} + +func defaultClient(logger log.Logger) *Client { + if logger == nil { + logger = log.NewNopLogger() + } + return NewClient( + logger, + &http.Client{ + Transport: tracing.HTTPTripperware(logger, http.DefaultTransport), + }, + ) +} +// QueryInstant performs an instant query and returns results in model.Vector type. +func (c *Client) QueryInstant(ctx context.Context, base *url.URL, query string, t time.Time, opts QueryOptions) (model.Vector, []string, error) { params, err := url.ParseQuery(base.RawQuery) if err != nil { return nil, nil, errors.Wrapf(err, "parse raw query %s", base.RawQuery) @@ -301,7 +331,7 @@ func QueryInstant(ctx context.Context, logger log.Logger, base *url.URL, query s u.Path = path.Join(u.Path, "/api/v1/query") u.RawQuery = params.Encode() - level.Debug(logger).Log("msg", "querying instant", "url", u.String()) + level.Debug(c.logger).Log("msg", "querying instant", "url", u.String()) req, err := http.NewRequest("GET", u.String(), nil) if err != nil { @@ -310,14 +340,11 @@ func QueryInstant(ctx context.Context, logger log.Logger, base *url.URL, query s req = req.WithContext(ctx) - client := &http.Client{ - Transport: tracing.HTTPTripperware(logger, http.DefaultTransport), - } - resp, err := client.Do(req) + resp, err := c.httpClient.Do(req) if err != nil { return nil, nil, errors.Wrapf(err, "perform GET request against %s", u.String()) } - defer runutil.ExhaustCloseWithLogOnErr(logger, resp.Body, "query body") + defer runutil.ExhaustCloseWithLogOnErr(c.logger, resp.Body, "query body") // Decode only ResultType and load Result only as RawJson since we don't know // structure of the Result yet. @@ -369,9 +396,14 @@ func QueryInstant(ctx context.Context, logger log.Logger, base *url.URL, query s return vectorResult, m.Warnings, nil } +// QueryInstant performs an instant query using a default HTTP client and returns results in model.Vector type. +func QueryInstant(ctx context.Context, logger log.Logger, base *url.URL, query string, t time.Time, opts QueryOptions) (model.Vector, []string, error) { + return defaultClient(logger).QueryInstant(ctx, base, query, t, opts) +} + // PromqlQueryInstant performs instant query and returns results in promql.Vector type that is compatible with promql package. -func PromqlQueryInstant(ctx context.Context, logger log.Logger, base *url.URL, query string, t time.Time, opts QueryOptions) (promql.Vector, []string, error) { - vectorResult, warnings, err := QueryInstant(ctx, logger, base, query, t, opts) +func (c *Client) PromqlQueryInstant(ctx context.Context, base *url.URL, query string, t time.Time, opts QueryOptions) (promql.Vector, []string, error) { + vectorResult, warnings, err := c.QueryInstant(ctx, base, query, t, opts) if err != nil { return nil, nil, err } @@ -398,6 +430,11 @@ func PromqlQueryInstant(ctx context.Context, logger log.Logger, base *url.URL, q return vec, warnings, nil } +// PromqlQueryInstant performs instant query and returns results in promql.Vector type that is compatible with promql package. +func PromqlQueryInstant(ctx context.Context, logger log.Logger, base *url.URL, query string, t time.Time, opts QueryOptions) (promql.Vector, []string, error) { + return defaultClient(logger).PromqlQueryInstant(ctx, base, query, t, opts) +} + // Scalar response consists of array with mixed types so it needs to be // unmarshaled separately. func convertScalarJSONToVector(scalarJSONResult json.RawMessage) (model.Vector, error) { diff --git a/pkg/query/config.go b/pkg/query/config.go new file mode 100644 index 0000000000..c4c918b74e --- /dev/null +++ b/pkg/query/config.go @@ -0,0 +1,38 @@ +package query + +import ( + "gopkg.in/yaml.v2" + + http_util "github.com/thanos-io/thanos/pkg/http" +) + +type Config struct { + HTTPClientConfig http_util.ClientConfig `yaml:"http_config"` + EndpointsConfig http_util.EndpointsConfig `yaml:",inline"` +} + +func DefaultConfig() Config { + return Config{ + EndpointsConfig: http_util.EndpointsConfig{ + Scheme: "http", + StaticAddresses: []string{}, + FileSDConfigs: []http_util.FileSDConfig{}, + }, + } +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultConfig() + type plain Config + return unmarshal((*plain)(c)) +} + +// LoadConfigs loads a list of Config from YAML data. +func LoadConfigs(confYAML []byte) ([]Config, error) { + var queryCfg []Config + if err := yaml.UnmarshalStrict(confYAML, &queryCfg); err != nil { + return nil, err + } + return queryCfg, nil +} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 5f33f2bdba..549db5de61 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/alert" "github.com/thanos-io/thanos/pkg/cacheutil" + http_util "github.com/thanos-io/thanos/pkg/http" "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/cos" @@ -22,6 +23,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" + "github.com/thanos-io/thanos/pkg/query" storecache "github.com/thanos-io/thanos/pkg/store/cache" trclient "github.com/thanos-io/thanos/pkg/tracing/client" "github.com/thanos-io/thanos/pkg/tracing/elasticapm" @@ -87,11 +89,18 @@ func main() { } alertmgrCfg := alert.DefaultAlertmanagerConfig() - alertmgrCfg.FileSDConfigs = []alert.FileSDConfig{alert.FileSDConfig{}} + alertmgrCfg.EndpointsConfig.FileSDConfigs = []http_util.FileSDConfig{http_util.FileSDConfig{}} if err := generate(alert.AlertingConfig{Alertmanagers: []alert.AlertmanagerConfig{alertmgrCfg}}, "rule_alerting", *outputDir); err != nil { level.Error(logger).Log("msg", "failed to generate", "type", "rule_alerting", "err", err) os.Exit(1) } + + queryCfg := query.DefaultConfig() + queryCfg.EndpointsConfig.FileSDConfigs = []http_util.FileSDConfig{http_util.FileSDConfig{}} + if err := generate([]query.Config{queryCfg}, "rule_query", *outputDir); err != nil { + level.Error(logger).Log("msg", "failed to generate", "type", "rule_query", "err", err) + os.Exit(1) + } logger.Log("msg", "success") } diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 66b7a01870..757ff7a6d0 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -27,6 +27,7 @@ import ( "github.com/thanos-io/thanos/pkg/alert" http_util "github.com/thanos-io/thanos/pkg/http" "github.com/thanos-io/thanos/pkg/promclient" + "github.com/thanos-io/thanos/pkg/query" rapi "github.com/thanos-io/thanos/pkg/rule/api" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" @@ -71,15 +72,39 @@ func createRuleFiles(t *testing.T, dir string) { } } +func writeTargets(t *testing.T, path string, addrs ...string) { + t.Helper() + + var tgs []model.LabelSet + for _, a := range addrs { + tgs = append( + tgs, + model.LabelSet{ + model.LabelName(model.AddressLabel): model.LabelValue(a), + }, + ) + } + b, err := yaml.Marshal([]*targetgroup.Group{&targetgroup.Group{Targets: tgs}}) + testutil.Ok(t, err) + + testutil.Ok(t, ioutil.WriteFile(path+".tmp", b, 0660)) + testutil.Ok(t, os.Rename(path+".tmp", path)) +} + func serializeAlertingConfiguration(t *testing.T, cfg ...alert.AlertmanagerConfig) []byte { t.Helper() amCfg := alert.AlertingConfig{ Alertmanagers: cfg, } b, err := yaml.Marshal(&amCfg) - if err != nil { - t.Errorf("failed to serialize alerting configuration: %v", err) - } + testutil.Ok(t, err) + return b +} + +func serializeQueryConfiguration(t *testing.T, cfg ...query.Config) []byte { + t.Helper() + b, err := yaml.Marshal(&cfg) + testutil.Ok(t, err) return b } @@ -195,10 +220,12 @@ func TestRuleAlertmanagerHTTPClient(t *testing.T) { amCfg := serializeAlertingConfiguration( t, alert.AlertmanagerConfig{ - StaticAddresses: []string{srv1.Listener.Addr().String()}, - Scheme: "http", - Timeout: model.Duration(time.Second), - PathPrefix: "/prefix/", + EndpointsConfig: http_util.EndpointsConfig{ + StaticAddresses: []string{srv1.Listener.Addr().String()}, + Scheme: "http", + PathPrefix: "/prefix/", + }, + Timeout: model.Duration(time.Second), }, alert.AlertmanagerConfig{ HTTPClientConfig: http_util.ClientConfig{ @@ -207,9 +234,11 @@ func TestRuleAlertmanagerHTTPClient(t *testing.T) { }, BearerToken: "secret", }, - StaticAddresses: []string{srv2.Listener.Addr().String()}, - Scheme: "https", - Timeout: model.Duration(time.Second), + EndpointsConfig: http_util.EndpointsConfig{ + StaticAddresses: []string{srv2.Listener.Addr().String()}, + Scheme: "https", + }, + Timeout: model.Duration(time.Second), }, ) @@ -219,7 +248,16 @@ func TestRuleAlertmanagerHTTPClient(t *testing.T) { createRuleFiles(t, rulesDir) qAddr := a.New() - r := rule(a.New(), a.New(), rulesDir, amCfg, []address{qAddr}, nil) + queryCfg := serializeQueryConfiguration( + t, + query.Config{ + EndpointsConfig: http_util.EndpointsConfig{ + StaticAddresses: []string{qAddr.HostPort()}, + Scheme: "http", + }, + }, + ) + r := rule(a.New(), a.New(), rulesDir, amCfg, queryCfg) q := querier(qAddr, a.New(), []address{r.GRPC}, nil) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) @@ -263,13 +301,15 @@ func TestRuleAlertmanagerFileSD(t *testing.T) { amCfg := serializeAlertingConfiguration( t, alert.AlertmanagerConfig{ - FileSDConfigs: []alert.FileSDConfig{ - alert.FileSDConfig{ - Files: []string{filepath.Join(amDir, "*.yaml")}, - RefreshInterval: model.Duration(time.Hour), + EndpointsConfig: http_util.EndpointsConfig{ + FileSDConfigs: []http_util.FileSDConfig{ + http_util.FileSDConfig{ + Files: []string{filepath.Join(amDir, "*.yaml")}, + RefreshInterval: model.Duration(time.Hour), + }, }, + Scheme: "http", }, - Scheme: "http", Timeout: model.Duration(time.Second), }, ) @@ -280,7 +320,16 @@ func TestRuleAlertmanagerFileSD(t *testing.T) { createRuleFiles(t, rulesDir) qAddr := a.New() - r := rule(a.New(), a.New(), rulesDir, amCfg, []address{qAddr}, nil) + queryCfg := serializeQueryConfiguration( + t, + query.Config{ + EndpointsConfig: http_util.EndpointsConfig{ + StaticAddresses: []string{qAddr.HostPort()}, + Scheme: "http", + }, + }, + ) + r := rule(a.New(), a.New(), rulesDir, amCfg, queryCfg) q := querier(qAddr, a.New(), []address{r.GRPC}, nil) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) @@ -331,20 +380,7 @@ func TestRuleAlertmanagerFileSD(t *testing.T) { })) // Add the Alertmanager address to the file SD directory. - fileSDPath := filepath.Join(amDir, "targets.yaml") - b, err := yaml.Marshal([]*targetgroup.Group{ - &targetgroup.Group{ - Targets: []model.LabelSet{ - model.LabelSet{ - model.LabelName(model.AddressLabel): model.LabelValue(am.HTTP.HostPort()), - }, - }, - }, - }) - testutil.Ok(t, err) - - testutil.Ok(t, ioutil.WriteFile(fileSDPath+".tmp", b, 0660)) - testutil.Ok(t, os.Rename(fileSDPath+".tmp", fileSDPath)) + writeTargets(t, filepath.Join(amDir, "targets.yaml"), am.HTTP.HostPort()) // Verify that alerts are received by Alertmanager. testutil.Ok(t, runutil.Retry(5*time.Second, ctx.Done(), func() (err error) { @@ -373,21 +409,50 @@ func TestRule(t *testing.T) { amCfg := serializeAlertingConfiguration( t, alert.AlertmanagerConfig{ - StaticAddresses: []string{am.HTTP.HostPort()}, - Scheme: "http", - Timeout: model.Duration(time.Second), + EndpointsConfig: http_util.EndpointsConfig{ + StaticAddresses: []string{am.HTTP.HostPort()}, + Scheme: "http", + }, + Timeout: model.Duration(time.Second), }, ) qAddr := a.New() + queryCfg1 := serializeQueryConfiguration( + t, + query.Config{ + EndpointsConfig: http_util.EndpointsConfig{ + StaticAddresses: []string{qAddr.HostPort()}, + Scheme: "http", + }, + }, + ) + qDir, err := ioutil.TempDir("", "query") + defer os.RemoveAll(qDir) + testutil.Ok(t, err) + writeTargets(t, filepath.Join(qDir, "targets.yaml"), qAddr.HostPort()) + queryCfg2 := serializeQueryConfiguration( + t, + query.Config{ + EndpointsConfig: http_util.EndpointsConfig{ + FileSDConfigs: []http_util.FileSDConfig{ + http_util.FileSDConfig{ + Files: []string{filepath.Join(qDir, "*.yaml")}, + RefreshInterval: model.Duration(time.Hour), + }, + }, + Scheme: "http", + }, + }, + ) rulesDir, err := ioutil.TempDir("", "rules") defer os.RemoveAll(rulesDir) testutil.Ok(t, err) createRuleFiles(t, rulesDir) - r1 := rule(a.New(), a.New(), rulesDir, amCfg, []address{qAddr}, nil) - r2 := rule(a.New(), a.New(), rulesDir, amCfg, nil, []address{qAddr}) + r1 := rule(a.New(), a.New(), rulesDir, amCfg, queryCfg1) + r2 := rule(a.New(), a.New(), rulesDir, amCfg, queryCfg2) q := querier(qAddr, a.New(), []address{r1.GRPC, r2.GRPC}, nil) @@ -597,15 +662,26 @@ func (a *failingStoreAPI) LabelValues(context.Context, *storepb.LabelValuesReque func TestRulePartialResponse(t *testing.T) { a := newLocalAddresser() qAddr := a.New() + queryCfg := serializeQueryConfiguration( + t, + query.Config{ + EndpointsConfig: http_util.EndpointsConfig{ + StaticAddresses: []string{qAddr.HostPort()}, + Scheme: "http", + }, + }, + ) f := fakeStoreAPI(a.New(), &failingStoreAPI{}) am := alertManager(a.New()) amCfg := serializeAlertingConfiguration( t, alert.AlertmanagerConfig{ - StaticAddresses: []string{am.HTTP.HostPort()}, - Scheme: "http", - Timeout: model.Duration(time.Second), + EndpointsConfig: http_util.EndpointsConfig{ + StaticAddresses: []string{am.HTTP.HostPort()}, + Scheme: "http", + }, + Timeout: model.Duration(time.Second), }, ) @@ -613,7 +689,7 @@ func TestRulePartialResponse(t *testing.T) { defer os.RemoveAll(rulesDir) testutil.Ok(t, err) - r := rule(a.New(), a.New(), rulesDir, amCfg, []address{qAddr}, nil) + r := rule(a.New(), a.New(), rulesDir, amCfg, queryCfg) q := querier(qAddr, a.New(), []address{r.GRPC, f.GRPC}, nil) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) diff --git a/test/e2e/spinup_test.go b/test/e2e/spinup_test.go index 721eb70403..54b34597ce 100644 --- a/test/e2e/spinup_test.go +++ b/test/e2e/spinup_test.go @@ -305,11 +305,15 @@ receivers: } } -func rule(http, grpc address, ruleDir string, amCfg []byte, queryAddresses []address, queryFileSDAddresses []address) *serverScheduler { +func rule(http, grpc address, ruleDir string, amCfg []byte, queryCfg []byte) *serverScheduler { return &serverScheduler{ HTTP: http, GRPC: grpc, schedule: func(workDir string) (Exec, error) { + err := ioutil.WriteFile(filepath.Join(workDir, "query.cfg"), queryCfg, 0666) + if err != nil { + return nil, errors.Wrap(err, "creating query config for ruler") + } args := []string{ "rule", "--debug.name", fmt.Sprintf("rule-%s", http.Port), @@ -323,20 +327,10 @@ func rule(http, grpc address, ruleDir string, amCfg []byte, queryAddresses []add "--grpc-grace-period", "0s", "--http-address", http.HostPort(), "--log.level", "debug", + "--query.config-file", filepath.Join(workDir, "query.cfg"), "--query.sd-dns-interval", "5s", "--resend-delay", "5s", } - - for _, addr := range queryAddresses { - args = append(args, "--query", addr.HostPort()) - } - - if len(queryFileSDAddresses) > 0 { - if err := ioutil.WriteFile(filepath.Join(workDir, "filesd.json"), []byte(generateFileSD(queryFileSDAddresses)), 0666); err != nil { - return nil, errors.Wrap(err, "creating ruler filesd config") - } - args = append(args, "--query.sd-files", filepath.Join(workDir, "filesd.json")) - } return newCmdExec(exec.Command("thanos", args...)), nil }, } From 552ffa4c1a0d6cd67f62d51be42a2ebf1764fd53 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Thu, 9 Jan 2020 21:39:23 +0100 Subject: [PATCH 163/257] Fixed crash in the memcached servers selector when there's only 1 server (#1975) * Fixed crash in the memcached servers selector when there's only 1 server Signed-off-by: Marco Pracucci * Updated changelog Signed-off-by: Marco Pracucci * Assert expected error in tests Signed-off-by: Marco Pracucci --- CHANGELOG.md | 1 + pkg/cacheutil/memcached_server_selector.go | 5 +- .../memcached_server_selector_test.go | 48 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d915df7e9c..34af658e1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags - [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction +- [#1975](https://github.com/thanos-io/thanos/pull/1975) Store Gateway: fixed panic caused by memcached servers selector when there's 1 memcached node ### Fixed diff --git a/pkg/cacheutil/memcached_server_selector.go b/pkg/cacheutil/memcached_server_selector.go index 1318574045..37b693367c 100644 --- a/pkg/cacheutil/memcached_server_selector.go +++ b/pkg/cacheutil/memcached_server_selector.go @@ -74,9 +74,12 @@ func (s *MemcachedJumpHashSelector) PickServer(key string) (net.Addr, error) { return nil, memcache.ErrNoServers } if len(addrs) == 1 { + picked := addrs[0] + addrs = (addrs)[:0] addrsPool.Put(&addrs) - return (addrs)[0], nil + + return picked, nil } // Pick a server using the jump hash. diff --git a/pkg/cacheutil/memcached_server_selector_test.go b/pkg/cacheutil/memcached_server_selector_test.go index a215604ece..466db1c401 100644 --- a/pkg/cacheutil/memcached_server_selector_test.go +++ b/pkg/cacheutil/memcached_server_selector_test.go @@ -37,6 +37,54 @@ func TestNatSort(t *testing.T) { testutil.Equals(t, expected, input) } +func TestMemcachedJumpHashSelector_PickServer(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + tests := []struct { + addrs []string + key string + expectedAddr string + expectedErr error + }{ + { + addrs: []string{}, + key: "test-1", + expectedErr: memcache.ErrNoServers, + }, + { + addrs: []string{"127.0.0.1:11211"}, + key: "test-1", + expectedAddr: "127.0.0.1:11211", + }, + { + addrs: []string{"127.0.0.1:11211", "127.0.0.2:11211"}, + key: "test-1", + expectedAddr: "127.0.0.1:11211", + }, + { + addrs: []string{"127.0.0.1:11211", "127.0.0.2:11211"}, + key: "test-2", + expectedAddr: "127.0.0.2:11211", + }, + } + + s := MemcachedJumpHashSelector{} + + for _, test := range tests { + testutil.Ok(t, s.SetServers(test.addrs...)) + + actualAddr, err := s.PickServer(test.key) + + if test.expectedErr != nil { + testutil.Equals(t, test.expectedErr, err) + testutil.Equals(t, nil, actualAddr) + } else { + testutil.Ok(t, err) + testutil.Equals(t, test.expectedAddr, actualAddr.String()) + } + } +} + func TestMemcachedJumpHashSelector_Each_ShouldRespectServersOrdering(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() From b02e99d33c6d1edf594652cea0276cd91432dd58 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 10 Jan 2020 19:07:34 +0800 Subject: [PATCH 164/257] Enable Stale Bot (#1977) * enable stale bot Signed-off-by: Xiang Dai <764524258@qq.com> * Revert "Enabled stale bot. (#1795)" This reverts commit 5b7e33e81121a02485fd872ab2ee9436c4275050. Signed-off-by: Xiang Dai <764524258@qq.com> --- .github/stale.yaml | 16 ------------- .github/stale.yml | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 16 deletions(-) delete mode 100644 .github/stale.yaml create mode 100644 .github/stale.yml diff --git a/.github/stale.yaml b/.github/stale.yaml deleted file mode 100644 index 3d454553ea..0000000000 --- a/.github/stale.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Number of days of inactivity before an issue becomes stale -daysUntilStale: 90 -# Number of days of inactivity before a stale issue is closed -daysUntilClose: 14 -# Issues with these labels will never be considered stale -exemptLabels: - - pinned -# Label to use when marking an issue as stale -staleLabel: stale -# Comment to post when marking an issue as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. Use `pinned` label to avoid closing. -# Comment to post when closing a stale issue. Set to `false` to disable -closeComment: false \ No newline at end of file diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 0000000000..4bd5519c11 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,60 @@ +# Configuration for probot-stale - https://github.com/probot/stale + +# Number of days of inactivity before an Issue or Pull Request becomes stale +daysUntilStale: 30 + +# Number of days of inactivity before an Issue or Pull Request with the stale label is closed. +# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. +daysUntilClose: 7 + +# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) +onlyLabels: [] + +# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable +exemptLabels: + - "priority: P0" + +# Set to true to ignore issues in a project (defaults to false) +exemptProjects: false + +# Set to true to ignore issues in a milestone (defaults to false) +exemptMilestones: false + +# Set to true to ignore issues with an assignee (defaults to false) +exemptAssignees: false + +# Label to use when marking as stale +staleLabel: stale + +# Comment to post when marking as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. + +# Comment to post when removing the stale label. +# unmarkComment: > +# Your comment here. + +# Comment to post when closing a stale Issue or Pull Request. +# closeComment: > +# Your comment here. + +# Limit the number of actions per hour, from 1-30. Default is 30 +limitPerRun: 30 + +# Limit to only `issues` or `pulls` +# only: issues + +# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls': +# pulls: +# daysUntilStale: 30 +# markComment: > +# This pull request has been automatically marked as stale because it has not had +# recent activity. It will be closed if no further activity occurs. Thank you +# for your contributions. + +# issues: +# exemptLabels: +# - confirmed + From 9c968a8d4daa5d8d1c355eab175639fa9def6698 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 10 Jan 2020 13:15:15 +0100 Subject: [PATCH 165/257] *: fix tracing for alerts (#1978) Signed-off-by: Simon Pasquier --- cmd/thanos/rule.go | 1 + pkg/alert/alert.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 65430e7849..037bda25bc 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -452,6 +452,7 @@ func runRule( { sdr := alert.NewSender(logger, reg, alertmgrs) ctx, cancel := context.WithCancel(context.Background()) + ctx = tracing.ContextWithTracer(ctx, tracer) g.Add(func() error { for { diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go index 2387a2245c..18e57d6d81 100644 --- a/pkg/alert/alert.go +++ b/pkg/alert/alert.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/tracing" ) const ( @@ -304,6 +305,8 @@ func NewSender( // Send an alert batch to all given Alertmanager clients. // TODO(bwplotka): https://github.com/thanos-io/thanos/issues/660. func (s *Sender) Send(ctx context.Context, alerts []*Alert) { + span, ctx := tracing.StartSpan(ctx, "/send_alerts") + defer span.Finish() if len(alerts) == 0 { return } @@ -325,6 +328,8 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { level.Debug(s.logger).Log("msg", "sending alerts", "alertmanager", u.Host, "numAlerts", len(alerts)) start := time.Now() + span, ctx := tracing.StartSpan(ctx, "post_alerts HTTP[client]") + defer span.Finish() if err := am.postAlerts(ctx, *u, bytes.NewReader(b)); err != nil { level.Warn(s.logger).Log( "msg", "sending alerts failed", From 3cf366a66a9f978a7503fb6cf630a53adb56d702 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 10 Jan 2020 06:42:45 -0600 Subject: [PATCH 166/257] receive: use gRPC for forwarding messages (#1970) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * receive: use grpc to forward remote write requests Signed-off-by: Brett Jones * test/e2e: update receive e2e tests to use gRPC Signed-off-by: Lucas Servén Marín * CHANGELOG: add note about receive config Signed-off-by: Lucas Servén Marín Co-authored-by: Lucas Servén Marín --- CHANGELOG.md | 1 + cmd/thanos/query.go | 53 +--- cmd/thanos/receive.go | 16 +- pkg/extgrpc/client.go | 59 ++++ pkg/receive/handler.go | 187 ++++++++---- pkg/receive/handler_test.go | 133 +++++---- pkg/server/grpc/grpc.go | 13 + pkg/store/storepb/rpc.pb.go | 576 ++++++++++++++++++++++++++++++++---- pkg/store/storepb/rpc.proto | 24 +- pkg/store/tsdb.go | 6 + scripts/genproto.sh | 5 +- test/e2e/receive_test.go | 28 +- test/e2e/spinup_test.go | 4 +- 13 files changed, 864 insertions(+), 241 deletions(-) create mode 100644 pkg/extgrpc/client.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 34af658e1c..985ba42fa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags - [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction - [#1975](https://github.com/thanos-io/thanos/pull/1975) Store Gateway: fixed panic caused by memcached servers selector when there's 1 memcached node +- [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`. ### Fixed diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 2dd9c17663..349e9bf2ea 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -10,8 +10,6 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/oklog/run" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -21,9 +19,12 @@ import ( "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql" + kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/discovery/cache" "github.com/thanos-io/thanos/pkg/discovery/dns" + "github.com/thanos-io/thanos/pkg/extgrpc" "github.com/thanos-io/thanos/pkg/extprom" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" "github.com/thanos-io/thanos/pkg/prober" @@ -34,11 +35,7 @@ import ( httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/tls" - "github.com/thanos-io/thanos/pkg/tracing" "github.com/thanos-io/thanos/pkg/ui" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - kingpin "gopkg.in/alecthomas/kingpin.v2" ) // registerQuery registers a query command. @@ -165,48 +162,6 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { } } -func storeClientGRPCOpts(logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, secure bool, cert, key, caCert, serverName string) ([]grpc.DialOption, error) { - grpcMets := grpc_prometheus.NewClientMetrics() - grpcMets.EnableClientHandlingTimeHistogram( - grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), - ) - dialOpts := []grpc.DialOption{ - // We want to make sure that we can receive huge gRPC messages from storeAPI. - // On TCP level we can be fine, but the gRPC overhead for huge messages could be significant. - // Current limit is ~2GB. - // TODO(bplotka): Split sent chunks on store node per max 4MB chunks if needed. - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)), - grpc.WithUnaryInterceptor( - grpc_middleware.ChainUnaryClient( - grpcMets.UnaryClientInterceptor(), - tracing.UnaryClientInterceptor(tracer), - ), - ), - grpc.WithStreamInterceptor( - grpc_middleware.ChainStreamClient( - grpcMets.StreamClientInterceptor(), - tracing.StreamClientInterceptor(tracer), - ), - ), - } - - if reg != nil { - reg.MustRegister(grpcMets) - } - - if !secure { - return append(dialOpts, grpc.WithInsecure()), nil - } - - level.Info(logger).Log("msg", "enabling client to server TLS") - - tlsCfg, err := tls.NewClientConfig(logger, cert, key, caCert, serverName) - if err != nil { - return nil, err - } - return append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))), nil -} - // runQuery starts a server that exposes PromQL Query API. It is responsible for querying configured // store nodes, merging and duplicating the data to satisfy user query. func runQuery( @@ -251,7 +206,7 @@ func runQuery( }) reg.MustRegister(duplicatedStores) - dialOpts, err := storeClientGRPCOpts(logger, reg, tracer, secure, cert, key, caCert, serverName) + dialOpts, err := extgrpc.StoreClientGRPCOpts(logger, reg, tracer, secure, cert, key, caCert, serverName) if err != nil { return errors.Wrap(err, "building gRPC client") } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index c55d328364..ba552e67e3 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -15,9 +15,12 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage/tsdb" + kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/extgrpc" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/receive" @@ -27,7 +30,6 @@ import ( "github.com/thanos-io/thanos/pkg/shipper" "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/tls" - kingpin "gopkg.in/alecthomas/kingpin.v2" ) func registerReceive(m map[string]setupFunc, app *kingpin.Application) { @@ -189,6 +191,11 @@ func runReceive( if err != nil { return err } + dialOpts, err := extgrpc.StoreClientGRPCOpts(logger, reg, tracer, rwServerCert != "", rwClientCert, rwClientKey, rwClientServerCA, rwClientServerName) + if err != nil { + return err + } + webHandler := receive.NewHandler(log.With(logger, "component", "receive-handler"), &receive.Options{ ListenAddress: rwAddress, Registry: reg, @@ -199,6 +206,7 @@ func runReceive( Tracer: tracer, TLSConfig: rwTLSConfig, TLSClientConfig: rwTLSClientConfig, + DialOpts: dialOpts, }) statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) @@ -376,8 +384,12 @@ func runReceive( s.Shutdown(errors.New("reload hashrings")) } tsdbStore := store.NewTSDBStore(log.With(logger, "component", "thanos-tsdb-store"), nil, localStorage.Get(), comp, lset) + rw := store.ReadWriteTSDBStore{ + StoreServer: tsdbStore, + WriteableStoreServer: webHandler, + } - s = grpcserver.New(logger, &receive.UnRegisterer{Registerer: reg}, tracer, comp, tsdbStore, + s = grpcserver.NewReadWrite(logger, &receive.UnRegisterer{Registerer: reg}, tracer, comp, rw, grpcserver.WithListen(grpcBindAddr), grpcserver.WithGracePeriod(grpcGracePeriod), grpcserver.WithTLSConfig(tlsCfg), diff --git a/pkg/extgrpc/client.go b/pkg/extgrpc/client.go new file mode 100644 index 0000000000..2d97f9894b --- /dev/null +++ b/pkg/extgrpc/client.go @@ -0,0 +1,59 @@ +package extgrpc + +import ( + "math" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + opentracing "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/thanos-io/thanos/pkg/tls" + "github.com/thanos-io/thanos/pkg/tracing" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// StoreClientGRPCOpts creates gRPC dial options for connecting to a store client. +func StoreClientGRPCOpts(logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, secure bool, cert, key, caCert, serverName string) ([]grpc.DialOption, error) { + grpcMets := grpc_prometheus.NewClientMetrics() + grpcMets.EnableClientHandlingTimeHistogram( + grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), + ) + dialOpts := []grpc.DialOption{ + // We want to make sure that we can receive huge gRPC messages from storeAPI. + // On TCP level we can be fine, but the gRPC overhead for huge messages could be significant. + // Current limit is ~2GB. + // TODO(bplotka): Split sent chunks on store node per max 4MB chunks if needed. + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)), + grpc.WithUnaryInterceptor( + grpc_middleware.ChainUnaryClient( + grpcMets.UnaryClientInterceptor(), + tracing.UnaryClientInterceptor(tracer), + ), + ), + grpc.WithStreamInterceptor( + grpc_middleware.ChainStreamClient( + grpcMets.StreamClientInterceptor(), + tracing.StreamClientInterceptor(tracer), + ), + ), + } + + if reg != nil { + reg.MustRegister(grpcMets) + } + + if !secure { + return append(dialOpts, grpc.WithInsecure()), nil + } + + level.Info(logger).Log("msg", "enabling client to server TLS") + + tlsCfg, err := tls.NewClientConfig(logger, cert, key, caCert, serverName) + if err != nil { + return nil, err + } + return append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))), nil +} diff --git a/pkg/receive/handler.go b/pkg/receive/handler.go index 7d082a9de1..5ee91d1017 100644 --- a/pkg/receive/handler.go +++ b/pkg/receive/handler.go @@ -1,7 +1,6 @@ package receive import ( - "bytes" "context" "crypto/tls" "fmt" @@ -24,8 +23,13 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage" terrors "github.com/prometheus/prometheus/tsdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/tracing" ) @@ -39,23 +43,25 @@ const ( // conflictErr is returned whenever an operation fails due to any conflict-type error. var conflictErr = errors.New("conflict") +var errBadReplica = errors.New("replica count exceeds replication factor") + // Options for the web Handler. type Options struct { Writer *Writer ListenAddress string Registry prometheus.Registerer - Endpoint string TenantHeader string ReplicaHeader string + Endpoint string ReplicationFactor uint64 Tracer opentracing.Tracer TLSConfig *tls.Config TLSClientConfig *tls.Config + DialOpts []grpc.DialOption } // Handler serves a Prometheus remote write receiving HTTP endpoint. type Handler struct { - client *http.Client logger log.Logger writer *Writer router *route.Router @@ -64,6 +70,7 @@ type Handler struct { mtx sync.RWMutex hashring Hashring + peers *peerGroup // Metrics. forwardRequestsTotal *prometheus.CounterVec @@ -74,19 +81,12 @@ func NewHandler(logger log.Logger, o *Options) *Handler { logger = log.NewNopLogger() } - transport := http.DefaultTransport.(*http.Transport) - transport.TLSClientConfig = o.TLSClientConfig - client := &http.Client{Transport: transport} - if o.Tracer != nil { - client.Transport = tracing.HTTPTripperware(logger, client.Transport) - } - h := &Handler{ - client: client, logger: logger, writer: o.Writer, router: route.New(), options: o, + peers: newPeerGroup(o.DialOpts...), forwardRequestsTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "thanos_receive_forward_requests_total", @@ -109,7 +109,7 @@ func NewHandler(logger log.Logger, o *Options) *Handler { return ins.NewHandler(name, http.HandlerFunc(next)) } - h.router.Post("/api/v1/receive", instrf("receive", readyf(h.receive))) + h.router.Post("/api/v1/receive", instrf("receive", readyf(h.receiveHTTP))) return h } @@ -199,7 +199,35 @@ type replica struct { replicated bool } -func (h *Handler) receive(w http.ResponseWriter, r *http.Request) { +func (h *Handler) handleRequest(ctx context.Context, rep uint64, tenant string, wreq *prompb.WriteRequest) error { + // The replica value in the header is one-indexed, thus we need >. + if rep > h.options.ReplicationFactor { + return errBadReplica + } + + r := replica{ + n: rep, + replicated: rep != 0, + } + + // on-the-wire format is 1-indexed and in-code is 0-indexed so we decrement the value if it was already replicated. + if r.replicated { + r.n-- + } + + // Forward any time series as necessary. All time series + // destined for the local node will be written to the receiver. + // Time series will be replicated as necessary. + if err := h.forward(ctx, tenant, r, wreq); err != nil { + if countCause(err, isConflict) > 0 { + return conflictErr + } + return err + } + return nil +} + +func (h *Handler) receiveHTTP(w http.ResponseWriter, r *http.Request) { compressed, err := ioutil.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -219,34 +247,28 @@ func (h *Handler) receive(w http.ResponseWriter, r *http.Request) { return } - var rep replica - replicaRaw := r.Header.Get(h.options.ReplicaHeader) + rep := uint64(0) // If the header is empty, we assume the request is not yet replicated. - if replicaRaw != "" { - if rep.n, err = strconv.ParseUint(replicaRaw, 10, 64); err != nil { + if replicaRaw := r.Header.Get(h.options.ReplicaHeader); replicaRaw != "" { + if rep, err = strconv.ParseUint(replicaRaw, 10, 64); err != nil { http.Error(w, "could not parse replica header", http.StatusBadRequest) return } - rep.replicated = true - } - // The replica value in the header is zero-indexed, thus we need >=. - if rep.n >= h.options.ReplicationFactor { - http.Error(w, "replica count exceeds replication factor", http.StatusBadRequest) - return } tenant := r.Header.Get(h.options.TenantHeader) - // Forward any time series as necessary. All time series - // destined for the local node will be written to the receiver. - // Time series will be replicated as necessary. - if err := h.forward(r.Context(), tenant, rep, &wreq); err != nil { - if countCause(err, isConflict) > 0 { - http.Error(w, err.Error(), http.StatusConflict) - return - } - http.Error(w, err.Error(), http.StatusInternalServerError) + err = h.handleRequest(r.Context(), rep, tenant, &wreq) + switch err { + case nil: return + case conflictErr: + http.Error(w, err.Error(), http.StatusConflict) + case errBadReplica: + http.Error(w, err.Error(), http.StatusBadRequest) + default: + level.Error(h.logger).Log("err", err, "msg", "internal server error") + http.Error(w, err.Error(), http.StatusInternalServerError) } } @@ -352,20 +374,7 @@ func (h *Handler) parallelizeRequests(ctx context.Context, tenant string, replic } // Make a request to the specified endpoint. go func(endpoint string) { - buf, err := proto.Marshal(wreqs[endpoint]) - if err != nil { - level.Error(h.logger).Log("msg", "marshaling proto", "err", err, "endpoint", endpoint) - ec <- err - return - } - req, err := http.NewRequest("POST", endpoint, bytes.NewBuffer(snappy.Encode(nil, buf))) - if err != nil { - level.Error(h.logger).Log("msg", "creating request", "err", err, "endpoint", endpoint) - ec <- err - return - } - req.Header.Add(h.options.TenantHeader, tenant) - req.Header.Add(h.options.ReplicaHeader, strconv.FormatUint(replicas[endpoint].n, 10)) + var err error // Increment the counters as necessary now that // the requests will go out. @@ -377,25 +386,29 @@ func (h *Handler) parallelizeRequests(ctx context.Context, tenant string, replic h.forwardRequestsTotal.WithLabelValues("success").Inc() }() + cl, err := h.peers.get(ctx, endpoint) + if err != nil { + level.Error(h.logger).Log("msg", "failed to get peer connection to forward request", "err", err, "endpoint", endpoint) + ec <- err + return + } + // Create a span to track the request made to another receive node. span, ctx := tracing.StartSpan(ctx, "thanos_receive_forward") defer span.Finish() // Actually make the request against the endpoint // we determined should handle these time series. - var res *http.Response - res, err = h.client.Do(req.WithContext(ctx)) + _, err = cl.RemoteWrite(ctx, &storepb.WriteRequest{ + Timeseries: wreqs[endpoint].Timeseries, + Tenant: tenant, + Replica: int64(replicas[endpoint].n + 1), // increment replica since on-the-wire format is 1-indexed and 0 indicates unreplicated. + }) if err != nil { level.Error(h.logger).Log("msg", "forwarding request", "err", err, "endpoint", endpoint) ec <- err return } - if res.StatusCode != http.StatusOK { - err = errors.New(strconv.Itoa(res.StatusCode)) - level.Error(h.logger).Log("msg", "forwarding returned non-200 status", "err", err, "endpoint", endpoint) - ec <- err - return - } ec <- nil }(endpoint) } @@ -456,6 +469,21 @@ func (h *Handler) replicate(ctx context.Context, tenant string, wreq *prompb.Wri return errors.Wrap(err, "could not replicate write request") } +// RemoteWrite implements the gRPC remote write handler for storepb.WriteableStore. +func (h *Handler) RemoteWrite(ctx context.Context, r *storepb.WriteRequest) (*storepb.WriteResponse, error) { + err := h.handleRequest(ctx, uint64(r.Replica), r.Tenant, &prompb.WriteRequest{Timeseries: r.Timeseries}) + switch err { + case nil: + return &storepb.WriteResponse{}, nil + case conflictErr: + return nil, status.Error(codes.AlreadyExists, err.Error()) + case errBadReplica: + return nil, status.Error(codes.InvalidArgument, err.Error()) + default: + return nil, status.Error(codes.Internal, err.Error()) + } +} + // countCause counts the number of errors within the given error // whose causes satisfy the given function. // countCause will inspect the error's cause or, if the error is a MultiError, @@ -479,5 +507,54 @@ func isConflict(err error) bool { if err == nil { return false } - return err == conflictErr || err == storage.ErrDuplicateSampleForTimestamp || err == storage.ErrOutOfOrderSample || err == storage.ErrOutOfBounds || err.Error() == strconv.Itoa(http.StatusConflict) + return err == conflictErr || + err == storage.ErrDuplicateSampleForTimestamp || + err == storage.ErrOutOfOrderSample || + err == storage.ErrOutOfBounds || + err.Error() == strconv.Itoa(http.StatusConflict) || + status.Code(err) == codes.AlreadyExists +} + +func newPeerGroup(dialOpts ...grpc.DialOption) *peerGroup { + return &peerGroup{ + dialOpts: dialOpts, + cache: map[string]storepb.WriteableStoreClient{}, + m: sync.RWMutex{}, + dialer: grpc.DialContext, + } +} + +type peerGroup struct { + dialOpts []grpc.DialOption + cache map[string]storepb.WriteableStoreClient + m sync.RWMutex + + // dialer is used for testing. + dialer func(ctx context.Context, target string, opts ...grpc.DialOption) (conn *grpc.ClientConn, err error) +} + +func (p *peerGroup) get(ctx context.Context, addr string) (storepb.WriteableStoreClient, error) { + // use a RLock first to prevent blocking if we don't need to. + p.m.RLock() + c, ok := p.cache[addr] + p.m.RUnlock() + if ok { + return c, nil + } + + p.m.Lock() + defer p.m.Unlock() + // Make sure that another caller hasn't created the connection since obtaining the write lock. + c, ok = p.cache[addr] + if ok { + return c, nil + } + conn, err := p.dialer(ctx, addr, p.dialOpts...) + if err != nil { + return nil, errors.Wrap(err, "failed to dial peer") + } + + client := storepb.NewWriteableStoreClient(conn) + p.cache[addr] = client + return client, nil } diff --git a/pkg/receive/handler_test.go b/pkg/receive/handler_test.go index a040a21c80..6f58b84cec 100644 --- a/pkg/receive/handler_test.go +++ b/pkg/receive/handler_test.go @@ -2,6 +2,9 @@ package receive import ( "bytes" + "context" + "fmt" + "math/rand" "net/http" "net/http/httptest" "strconv" @@ -16,6 +19,9 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage" terrors "github.com/prometheus/prometheus/tsdb/errors" + "google.golang.org/grpc" + + "github.com/thanos-io/thanos/pkg/store/storepb" ) func TestCountCause(t *testing.T) { @@ -129,14 +135,27 @@ func TestCountCause(t *testing.T) { } } -func newHandlerHashring(appendables []*fakeAppendable, replicationFactor uint64) ([]*Handler, Hashring, func()) { +func newHandlerHashring(appendables []*fakeAppendable, replicationFactor uint64) ([]*Handler, Hashring) { cfg := []HashringConfig{ { Hashring: "test", }, } - var closers []func() var handlers []*Handler + // create a fake peer group where we manually fill the cache with fake addresses pointed to our handlers + // This removes the network from the tests and creates a more consistent testing harness. + peers := &peerGroup{ + dialOpts: nil, + m: sync.RWMutex{}, + cache: map[string]storepb.WriteableStoreClient{}, + dialer: func(context.Context, string, ...grpc.DialOption) (*grpc.ClientConn, error) { + // dialer should never be called since we are creating fake clients with fake addresses + // this protects against some leaking test that may attempt to dial random IP addresses + // which may pose a security risk. + return nil, errors.New("unexpected dial called in testing") + }, + } + for i := range appendables { h := NewHandler(nil, &Options{ TenantHeader: DefaultTenantHeader, @@ -145,21 +164,17 @@ func newHandlerHashring(appendables []*fakeAppendable, replicationFactor uint64) Writer: NewWriter(log.NewNopLogger(), appendables[i]), }) handlers = append(handlers, h) - ts := httptest.NewServer(h.router) - closers = append(closers, ts.Close) - h.options.Endpoint = ts.URL + "/api/v1/receive" + h.peers = peers + addr := randomAddr() + h.options.Endpoint = addr cfg[0].Endpoints = append(cfg[0].Endpoints, h.options.Endpoint) + peers.cache[addr] = &fakeRemoteWriteGRPCServer{h: h} } hashring := newMultiHashring(cfg) for _, h := range handlers { h.Hashring(hashring) } - close := func() { - for _, closer := range closers { - closer() - } - } - return handlers, hashring, close + return handlers, hashring } func TestReceive(t *testing.T) { @@ -454,53 +469,54 @@ func TestReceive(t *testing.T) { }, }, } { - handlers, hashring, close := newHandlerHashring(tc.appendables, tc.replicationFactor) - defer close() - tenant := "test" - // Test from the point of view of every node - // so that we know status code does not depend - // on which node is erroring and which node is receiving. - for i, handler := range handlers { - // Test that the correct status is returned. - status, err := makeRequest(handler, tenant, tc.wreq) - if err != nil { - t.Fatalf("test case %q, handler %d: unexpectedly failed making HTTP request: %v", tc.name, tc.status, err) - } - if status != tc.status { - t.Errorf("test case %q, handler %d: got unexpected HTTP status code: expected %d, got %d", tc.name, i, tc.status, status) - } - } - // Test that each time series is stored - // the correct amount of times in each fake DB. - for _, ts := range tc.wreq.Timeseries { - lset := make(labels.Labels, len(ts.Labels)) - for j := range ts.Labels { - lset[j] = labels.Label{ - Name: ts.Labels[j].Name, - Value: ts.Labels[j].Value, + t.Run(tc.name, func(t *testing.T) { + handlers, hashring := newHandlerHashring(tc.appendables, tc.replicationFactor) + tenant := "test" + // Test from the point of view of every node + // so that we know status code does not depend + // on which node is erroring and which node is receiving. + for i, handler := range handlers { + // Test that the correct status is returned. + status, err := makeRequest(handler, tenant, tc.wreq) + if err != nil { + t.Fatalf("handler %d: unexpectedly failed making HTTP request: %v", tc.status, err) + } + if status != tc.status { + t.Errorf("handler %d: got unexpected HTTP status code: expected %d, got %d", i, tc.status, status) } } - for j, a := range tc.appendables { - var expected int - got := uint64(len(a.appender.(*fakeAppender).samples[lset.String()])) - if a.appenderErr == nil && endpointHit(t, hashring, tc.replicationFactor, handlers[j].options.Endpoint, tenant, &ts) { - // We have len(handlers) copies of each sample because the test case - // is run once for each handler and they all use the same appender. - expected = len(handlers) * len(ts.Samples) + // Test that each time series is stored + // the correct amount of times in each fake DB. + for _, ts := range tc.wreq.Timeseries { + lset := make(labels.Labels, len(ts.Labels)) + for j := range ts.Labels { + lset[j] = labels.Label{ + Name: ts.Labels[j].Name, + Value: ts.Labels[j].Value, + } } - if uint64(expected) != got { - t.Errorf("test case %q, labels %q: expected %d samples, got %d", tc.name, lset.String(), expected, got) + for j, a := range tc.appendables { + var expected int + n := a.appender.(*fakeAppender).samples[lset.String()] + got := uint64(len(n)) + if a.appenderErr == nil && endpointHit(t, hashring, tc.replicationFactor, handlers[j].options.Endpoint, tenant, &ts) { + // We have len(handlers) copies of each sample because the test case + // is run once for each handler and they all use the same appender. + expected = len(handlers) * len(ts.Samples) + } + if uint64(expected) != got { + t.Errorf("handler: %d, labels %q: expected %d samples, got %d", j, lset.String(), expected, got) + } } } - } + }) } } // endpointHit is a helper to determine if a given endpoint in a hashring would be selected // for a given time series, tenant, and replication factor. func endpointHit(t *testing.T, h Hashring, rf uint64, endpoint, tenant string, timeSeries *prompb.TimeSeries) bool { - var i uint64 - for i = 0; i < rf; i++ { + for i := uint64(0); i < rf; i++ { e, err := h.GetN(tenant, timeSeries, i) if err != nil { t.Fatalf("got unexpected error querying hashring: %v", err) @@ -539,9 +555,22 @@ func makeRequest(h *Handler, tenant string, wreq *prompb.WriteRequest) (int, err return 0, errors.Wrap(err, "create request") } req.Header.Add(h.options.TenantHeader, tenant) - res, err := h.client.Do(req) - if err != nil { - return 0, errors.Wrap(err, "make request") - } - return res.StatusCode, nil + + rec := httptest.NewRecorder() + h.receiveHTTP(rec, req) + rec.Flush() + + return rec.Code, nil +} + +func randomAddr() string { + return fmt.Sprintf("http://%d.%d.%d.%d:%d", rand.Intn(256), rand.Intn(256), rand.Intn(256), rand.Intn(256), rand.Intn(35000)+30000) +} + +type fakeRemoteWriteGRPCServer struct { + h storepb.WriteableStoreServer +} + +func (f *fakeRemoteWriteGRPCServer) RemoteWrite(ctx context.Context, in *storepb.WriteRequest, opts ...grpc.CallOption) (*storepb.WriteResponse, error) { + return f.h.RemoteWrite(ctx, in) } diff --git a/pkg/server/grpc/grpc.go b/pkg/server/grpc/grpc.go index 21416b8fda..38b1a1a3ac 100644 --- a/pkg/server/grpc/grpc.go +++ b/pkg/server/grpc/grpc.go @@ -127,3 +127,16 @@ func (s *Server) Shutdown(err error) { cancel() } } + +// ReadWriteStoreServer is a StoreServer and a WriteableStoreServer. +type ReadWriteStoreServer interface { + storepb.StoreServer + storepb.WriteableStoreServer +} + +// NewReadWrite creates a new server that can be written to. +func NewReadWrite(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, comp component.Component, storeSrv ReadWriteStoreServer, opts ...Option) *Server { + s := New(logger, reg, tracer, comp, storeSrv, opts...) + storepb.RegisterWriteableStoreServer(s.srv, storeSrv) + return s +} diff --git a/pkg/store/storepb/rpc.pb.go b/pkg/store/storepb/rpc.pb.go index 77b5b5fa03..63f1ab4974 100644 --- a/pkg/store/storepb/rpc.pb.go +++ b/pkg/store/storepb/rpc.pb.go @@ -12,6 +12,7 @@ import ( _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" + prompb "github.com/prometheus/prometheus/prompb" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -136,6 +137,81 @@ func (Aggr) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{2} } +type WriteResponse struct { +} + +func (m *WriteResponse) Reset() { *m = WriteResponse{} } +func (m *WriteResponse) String() string { return proto.CompactTextString(m) } +func (*WriteResponse) ProtoMessage() {} +func (*WriteResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_77a6da22d6a3feb1, []int{0} +} +func (m *WriteResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *WriteResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_WriteResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *WriteResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_WriteResponse.Merge(m, src) +} +func (m *WriteResponse) XXX_Size() int { + return m.Size() +} +func (m *WriteResponse) XXX_DiscardUnknown() { + xxx_messageInfo_WriteResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_WriteResponse proto.InternalMessageInfo + +type WriteRequest struct { + Timeseries []prompb.TimeSeries `protobuf:"bytes,1,rep,name=timeseries,proto3" json:"timeseries"` + Tenant string `protobuf:"bytes,2,opt,name=tenant,proto3" json:"tenant,omitempty"` + Replica int64 `protobuf:"varint,3,opt,name=replica,proto3" json:"replica,omitempty"` +} + +func (m *WriteRequest) Reset() { *m = WriteRequest{} } +func (m *WriteRequest) String() string { return proto.CompactTextString(m) } +func (*WriteRequest) ProtoMessage() {} +func (*WriteRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_77a6da22d6a3feb1, []int{1} +} +func (m *WriteRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *WriteRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_WriteRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *WriteRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_WriteRequest.Merge(m, src) +} +func (m *WriteRequest) XXX_Size() int { + return m.Size() +} +func (m *WriteRequest) XXX_DiscardUnknown() { + xxx_messageInfo_WriteRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_WriteRequest proto.InternalMessageInfo + type InfoRequest struct { } @@ -143,7 +219,7 @@ func (m *InfoRequest) Reset() { *m = InfoRequest{} } func (m *InfoRequest) String() string { return proto.CompactTextString(m) } func (*InfoRequest) ProtoMessage() {} func (*InfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{0} + return fileDescriptor_77a6da22d6a3feb1, []int{2} } func (m *InfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -186,7 +262,7 @@ func (m *InfoResponse) Reset() { *m = InfoResponse{} } func (m *InfoResponse) String() string { return proto.CompactTextString(m) } func (*InfoResponse) ProtoMessage() {} func (*InfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{1} + return fileDescriptor_77a6da22d6a3feb1, []int{3} } func (m *InfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -223,7 +299,7 @@ func (m *LabelSet) Reset() { *m = LabelSet{} } func (m *LabelSet) String() string { return proto.CompactTextString(m) } func (*LabelSet) ProtoMessage() {} func (*LabelSet) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{2} + return fileDescriptor_77a6da22d6a3feb1, []int{4} } func (m *LabelSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -270,7 +346,7 @@ func (m *SeriesRequest) Reset() { *m = SeriesRequest{} } func (m *SeriesRequest) String() string { return proto.CompactTextString(m) } func (*SeriesRequest) ProtoMessage() {} func (*SeriesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{3} + return fileDescriptor_77a6da22d6a3feb1, []int{5} } func (m *SeriesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -310,7 +386,7 @@ func (m *SeriesResponse) Reset() { *m = SeriesResponse{} } func (m *SeriesResponse) String() string { return proto.CompactTextString(m) } func (*SeriesResponse) ProtoMessage() {} func (*SeriesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{4} + return fileDescriptor_77a6da22d6a3feb1, []int{6} } func (m *SeriesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -394,7 +470,7 @@ func (m *LabelNamesRequest) Reset() { *m = LabelNamesRequest{} } func (m *LabelNamesRequest) String() string { return proto.CompactTextString(m) } func (*LabelNamesRequest) ProtoMessage() {} func (*LabelNamesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{5} + return fileDescriptor_77a6da22d6a3feb1, []int{7} } func (m *LabelNamesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -432,7 +508,7 @@ func (m *LabelNamesResponse) Reset() { *m = LabelNamesResponse{} } func (m *LabelNamesResponse) String() string { return proto.CompactTextString(m) } func (*LabelNamesResponse) ProtoMessage() {} func (*LabelNamesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{6} + return fileDescriptor_77a6da22d6a3feb1, []int{8} } func (m *LabelNamesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -472,7 +548,7 @@ func (m *LabelValuesRequest) Reset() { *m = LabelValuesRequest{} } func (m *LabelValuesRequest) String() string { return proto.CompactTextString(m) } func (*LabelValuesRequest) ProtoMessage() {} func (*LabelValuesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{7} + return fileDescriptor_77a6da22d6a3feb1, []int{9} } func (m *LabelValuesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -510,7 +586,7 @@ func (m *LabelValuesResponse) Reset() { *m = LabelValuesResponse{} } func (m *LabelValuesResponse) String() string { return proto.CompactTextString(m) } func (*LabelValuesResponse) ProtoMessage() {} func (*LabelValuesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{8} + return fileDescriptor_77a6da22d6a3feb1, []int{10} } func (m *LabelValuesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -543,6 +619,8 @@ func init() { proto.RegisterEnum("thanos.StoreType", StoreType_name, StoreType_value) proto.RegisterEnum("thanos.PartialResponseStrategy", PartialResponseStrategy_name, PartialResponseStrategy_value) proto.RegisterEnum("thanos.Aggr", Aggr_name, Aggr_value) + proto.RegisterType((*WriteResponse)(nil), "thanos.WriteResponse") + proto.RegisterType((*WriteRequest)(nil), "thanos.WriteRequest") proto.RegisterType((*InfoRequest)(nil), "thanos.InfoRequest") proto.RegisterType((*InfoResponse)(nil), "thanos.InfoResponse") proto.RegisterType((*LabelSet)(nil), "thanos.LabelSet") @@ -557,58 +635,66 @@ func init() { func init() { proto.RegisterFile("rpc.proto", fileDescriptor_77a6da22d6a3feb1) } var fileDescriptor_77a6da22d6a3feb1 = []byte{ - // 814 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x4f, 0x6f, 0xe3, 0x44, - 0x14, 0xf7, 0xd8, 0x89, 0x13, 0xbf, 0x6c, 0x2b, 0xef, 0x34, 0xbb, 0xeb, 0x1a, 0x29, 0xad, 0x2c, - 0x21, 0x45, 0x05, 0x65, 0x21, 0x08, 0x10, 0xdc, 0x92, 0xac, 0x57, 0x1b, 0xb1, 0x4d, 0x60, 0x92, - 0x6c, 0xf8, 0x73, 0x08, 0x4e, 0x3b, 0xb8, 0xd6, 0x3a, 0xb6, 0xf1, 0x38, 0xb4, 0xbd, 0xf2, 0x09, - 0xb8, 0xf2, 0x1d, 0xf8, 0x16, 0x5c, 0x7a, 0xdc, 0x23, 0x5c, 0x10, 0xb4, 0x5f, 0x04, 0x79, 0x3c, - 0x4e, 0x62, 0x68, 0x2b, 0xad, 0x72, 0x9b, 0xf7, 0xfb, 0xbd, 0x79, 0x6f, 0xde, 0xef, 0xbd, 0x99, - 0x01, 0x2d, 0x8e, 0x4e, 0x5a, 0x51, 0x1c, 0x26, 0x21, 0x56, 0x93, 0x33, 0x27, 0x08, 0x99, 0x59, - 0x4b, 0x2e, 0x23, 0xca, 0x32, 0xd0, 0xac, 0xbb, 0xa1, 0x1b, 0xf2, 0xe5, 0xd3, 0x74, 0x95, 0xa1, - 0xd6, 0x0e, 0xd4, 0xfa, 0xc1, 0x0f, 0x21, 0xa1, 0x3f, 0x2e, 0x29, 0x4b, 0xac, 0x3f, 0x11, 0x3c, - 0xc8, 0x6c, 0x16, 0x85, 0x01, 0xa3, 0xf8, 0x3d, 0x50, 0x7d, 0x67, 0x4e, 0x7d, 0x66, 0xa0, 0x43, - 0xa5, 0x59, 0x6b, 0xef, 0xb4, 0xb2, 0xd8, 0xad, 0x97, 0x29, 0xda, 0x2d, 0x5d, 0xfd, 0x75, 0x20, - 0x11, 0xe1, 0x82, 0xf7, 0xa1, 0xba, 0xf0, 0x82, 0x59, 0xe2, 0x2d, 0xa8, 0x21, 0x1f, 0xa2, 0xa6, - 0x42, 0x2a, 0x0b, 0x2f, 0x18, 0x7b, 0x0b, 0xca, 0x29, 0xe7, 0x22, 0xa3, 0x14, 0x41, 0x39, 0x17, - 0x9c, 0x7a, 0x0a, 0x1a, 0x4b, 0xc2, 0x98, 0x8e, 0x2f, 0x23, 0x6a, 0x94, 0x0e, 0x51, 0x73, 0xb7, - 0xfd, 0x30, 0xcf, 0x32, 0xca, 0x09, 0xb2, 0xf6, 0xc1, 0x1f, 0x03, 0xf0, 0x84, 0x33, 0x46, 0x13, - 0x66, 0x94, 0xf9, 0xb9, 0xf4, 0xc2, 0xb9, 0x46, 0x34, 0x11, 0x47, 0xd3, 0x7c, 0x61, 0x33, 0xeb, - 0x53, 0xa8, 0xe6, 0xe4, 0x5b, 0x95, 0x65, 0xfd, 0xaa, 0xc0, 0xce, 0x88, 0xc6, 0x1e, 0x65, 0x42, - 0xa6, 0x42, 0xa1, 0xe8, 0xee, 0x42, 0xe5, 0x62, 0xa1, 0x9f, 0xa4, 0x54, 0x72, 0x72, 0x46, 0x63, - 0x66, 0x28, 0x3c, 0x6d, 0xbd, 0x90, 0xf6, 0x38, 0x23, 0x45, 0xf6, 0x95, 0x2f, 0x6e, 0xc3, 0xa3, - 0x34, 0x64, 0x4c, 0x59, 0xe8, 0x2f, 0x13, 0x2f, 0x0c, 0x66, 0xe7, 0x5e, 0x70, 0x1a, 0x9e, 0x73, - 0xb1, 0x14, 0xb2, 0xb7, 0x70, 0x2e, 0xc8, 0x8a, 0x9b, 0x72, 0x0a, 0xbf, 0x0f, 0xe0, 0xb8, 0x6e, - 0x4c, 0x5d, 0x27, 0xa1, 0x99, 0x46, 0xbb, 0xed, 0x07, 0x79, 0xb6, 0x8e, 0xeb, 0xc6, 0x64, 0x83, - 0xc7, 0x9f, 0xc3, 0x7e, 0xe4, 0xc4, 0x89, 0xe7, 0xf8, 0x69, 0x16, 0xde, 0xf9, 0xd9, 0xa9, 0xc7, - 0x9c, 0xb9, 0x4f, 0x4f, 0x0d, 0xf5, 0x10, 0x35, 0xab, 0xe4, 0x89, 0x70, 0xc8, 0x27, 0xe3, 0x99, - 0xa0, 0xf1, 0x77, 0xb7, 0xec, 0x65, 0x49, 0xec, 0x24, 0xd4, 0xbd, 0x34, 0x2a, 0xbc, 0x9d, 0x07, - 0x79, 0xe2, 0x2f, 0x8b, 0x31, 0x46, 0xc2, 0xed, 0x7f, 0xc1, 0x73, 0x02, 0x1f, 0x40, 0x8d, 0xbd, - 0xf6, 0xa2, 0xd9, 0xc9, 0xd9, 0x32, 0x78, 0xcd, 0x8c, 0x2a, 0x3f, 0x0a, 0xa4, 0x50, 0x8f, 0x23, - 0xd6, 0xf7, 0xb0, 0x9b, 0xb7, 0x46, 0x4c, 0x6c, 0x13, 0x54, 0xc6, 0x11, 0xde, 0x99, 0x5a, 0x7b, - 0x77, 0x35, 0x4b, 0x1c, 0x7d, 0x21, 0x11, 0xc1, 0x63, 0x13, 0x2a, 0xe7, 0x4e, 0x1c, 0x78, 0x81, - 0xcb, 0x3b, 0xa5, 0xbd, 0x90, 0x48, 0x0e, 0x74, 0xab, 0xa0, 0xc6, 0x94, 0x2d, 0xfd, 0xc4, 0xfa, - 0x0d, 0xc1, 0x43, 0xde, 0x9e, 0x81, 0xb3, 0x58, 0x4f, 0xc0, 0xbd, 0x8a, 0xa1, 0x2d, 0x14, 0x93, - 0xb7, 0x53, 0xcc, 0x7a, 0x0e, 0x78, 0xf3, 0xb4, 0x42, 0x94, 0x3a, 0x94, 0x83, 0x14, 0xe0, 0xe3, - 0xae, 0x91, 0xcc, 0xc0, 0x26, 0x54, 0x45, 0xbd, 0xcc, 0x90, 0x39, 0xb1, 0xb2, 0xad, 0xdf, 0x91, - 0x08, 0xf4, 0xca, 0xf1, 0x97, 0xeb, 0xba, 0xeb, 0x50, 0xe6, 0xb7, 0x82, 0xd7, 0xa8, 0x91, 0xcc, - 0xb8, 0x5f, 0x0d, 0x79, 0x0b, 0x35, 0x94, 0x2d, 0xd5, 0xe8, 0xc3, 0x5e, 0xa1, 0x08, 0x21, 0xc7, - 0x63, 0x50, 0x7f, 0xe2, 0x88, 0xd0, 0x43, 0x58, 0xf7, 0x09, 0x72, 0x44, 0x40, 0x5b, 0xbd, 0x46, - 0xb8, 0x06, 0x95, 0xc9, 0xe0, 0x8b, 0xc1, 0x70, 0x3a, 0xd0, 0x25, 0xac, 0x41, 0xf9, 0xab, 0x89, - 0x4d, 0xbe, 0xd1, 0x11, 0xae, 0x42, 0x89, 0x4c, 0x5e, 0xda, 0xba, 0x9c, 0x7a, 0x8c, 0xfa, 0xcf, - 0xec, 0x5e, 0x87, 0xe8, 0x4a, 0xea, 0x31, 0x1a, 0x0f, 0x89, 0xad, 0x97, 0x52, 0x9c, 0xd8, 0x3d, - 0xbb, 0xff, 0xca, 0xd6, 0xcb, 0x47, 0x2d, 0x78, 0x72, 0x47, 0x49, 0x69, 0xa4, 0x69, 0x87, 0x88, - 0xf0, 0x9d, 0xee, 0x90, 0x8c, 0x75, 0x74, 0xd4, 0x85, 0x52, 0x7a, 0x77, 0x71, 0x05, 0x14, 0xd2, - 0x99, 0x66, 0x5c, 0x6f, 0x38, 0x19, 0x8c, 0x75, 0x94, 0x62, 0xa3, 0xc9, 0xb1, 0x2e, 0xa7, 0x8b, - 0xe3, 0xfe, 0x40, 0x57, 0xf8, 0xa2, 0xf3, 0x75, 0x96, 0x93, 0x7b, 0xd9, 0x44, 0x2f, 0xb7, 0x7f, - 0x96, 0xa1, 0xcc, 0x0b, 0xc1, 0x1f, 0x42, 0x29, 0x7d, 0xeb, 0xf1, 0x5e, 0x2e, 0xef, 0xc6, 0x4f, - 0x60, 0xd6, 0x8b, 0xa0, 0x10, 0xee, 0x33, 0x50, 0xb3, 0x6b, 0x84, 0x1f, 0x15, 0xaf, 0x55, 0xbe, - 0xed, 0xf1, 0x7f, 0xe1, 0x6c, 0xe3, 0x07, 0x08, 0xf7, 0x00, 0xd6, 0x83, 0x89, 0xf7, 0x0b, 0x2f, - 0xdf, 0xe6, 0xd5, 0x32, 0xcd, 0xdb, 0x28, 0x91, 0xff, 0x39, 0xd4, 0x36, 0xfa, 0x89, 0x8b, 0xae, - 0x85, 0x49, 0x35, 0xdf, 0xb9, 0x95, 0xcb, 0xe2, 0x74, 0xdf, 0xbd, 0xfa, 0xa7, 0x21, 0x5d, 0x5d, - 0x37, 0xd0, 0x9b, 0xeb, 0x06, 0xfa, 0xfb, 0xba, 0x81, 0x7e, 0xb9, 0x69, 0x48, 0x6f, 0x6e, 0x1a, - 0xd2, 0x1f, 0x37, 0x0d, 0xe9, 0xdb, 0x0a, 0xff, 0x6b, 0xa2, 0xf9, 0x5c, 0xe5, 0x9f, 0xe4, 0x47, - 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x34, 0x17, 0xba, 0x5c, 0x07, 0x00, 0x00, + // 934 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcb, 0x6e, 0x23, 0x45, + 0x14, 0x75, 0xbb, 0xfd, 0xbc, 0x4e, 0x4c, 0x4f, 0xc5, 0xc9, 0x74, 0x8c, 0xe4, 0x58, 0x2d, 0x21, + 0x59, 0x01, 0x39, 0xe0, 0x11, 0x20, 0xd0, 0x6c, 0x6c, 0x8f, 0x47, 0x63, 0x31, 0x71, 0xa0, 0x6c, + 0x8f, 0x79, 0x2c, 0x4c, 0xdb, 0x29, 0xda, 0xad, 0x71, 0x3f, 0xe8, 0x2a, 0x93, 0x64, 0xc3, 0x82, + 0x2f, 0x60, 0xcb, 0x3f, 0xf0, 0x17, 0x6c, 0xb2, 0x9c, 0x25, 0x6c, 0x10, 0x24, 0x3f, 0x82, 0xaa, + 0xba, 0xda, 0xee, 0x0e, 0x49, 0xa4, 0x51, 0x76, 0x75, 0xcf, 0xb9, 0xbe, 0x8f, 0x53, 0xf7, 0x76, + 0x19, 0x8a, 0x81, 0x3f, 0x6f, 0xfa, 0x81, 0xc7, 0x3c, 0x94, 0x63, 0x0b, 0xd3, 0xf5, 0x68, 0xb5, + 0xc4, 0x2e, 0x7c, 0x42, 0x43, 0xb0, 0x5a, 0xb1, 0x3c, 0xcb, 0x13, 0xc7, 0x23, 0x7e, 0x92, 0xe8, + 0x13, 0xcb, 0x66, 0x8b, 0xd5, 0xac, 0x39, 0xf7, 0x9c, 0x23, 0x3f, 0xf0, 0x1c, 0xc2, 0x16, 0x64, + 0x45, 0x6f, 0x1e, 0xfd, 0xd9, 0x51, 0x2c, 0x94, 0xf1, 0x0e, 0x6c, 0x4f, 0x02, 0x9b, 0x11, 0x4c, + 0xa8, 0xef, 0xb9, 0x94, 0x18, 0x3f, 0xc3, 0x96, 0x04, 0x7e, 0x5c, 0x11, 0xca, 0xd0, 0x53, 0x00, + 0x66, 0x3b, 0x84, 0x92, 0xc0, 0x26, 0x54, 0x57, 0xea, 0x6a, 0xa3, 0xd4, 0xda, 0x6b, 0x6e, 0x82, + 0x36, 0x47, 0xb6, 0x43, 0x86, 0x82, 0xed, 0x64, 0x2e, 0xff, 0x3e, 0x48, 0xe1, 0x98, 0x3f, 0xda, + 0x83, 0x1c, 0x23, 0xae, 0xe9, 0x32, 0x3d, 0x5d, 0x57, 0x1a, 0x45, 0x2c, 0x2d, 0xa4, 0x43, 0x3e, + 0x20, 0xfe, 0xd2, 0x9e, 0x9b, 0xba, 0x5a, 0x57, 0x1a, 0x2a, 0x8e, 0x4c, 0x63, 0x1b, 0x4a, 0x7d, + 0xf7, 0x07, 0x4f, 0xa6, 0x37, 0xfe, 0x52, 0x60, 0x2b, 0xb4, 0xc3, 0xfa, 0xd0, 0xfb, 0x90, 0x5b, + 0x9a, 0x33, 0xb2, 0x8c, 0x6a, 0xd9, 0x6e, 0x86, 0x0a, 0x35, 0x5f, 0x72, 0x54, 0x96, 0x20, 0x5d, + 0xd0, 0x3e, 0x14, 0x1c, 0xdb, 0x9d, 0xf2, 0x82, 0x44, 0x01, 0x2a, 0xce, 0x3b, 0xb6, 0xcb, 0x2b, + 0x16, 0x94, 0x79, 0x1e, 0x52, 0xb2, 0x04, 0xc7, 0x3c, 0x17, 0xd4, 0x11, 0x14, 0x29, 0xf3, 0x02, + 0x32, 0xba, 0xf0, 0x89, 0x9e, 0xa9, 0x2b, 0x8d, 0x72, 0xeb, 0x51, 0x94, 0x65, 0x18, 0x11, 0x78, + 0xe3, 0x83, 0x3e, 0x06, 0x10, 0x09, 0xa7, 0x94, 0x30, 0xaa, 0x67, 0x45, 0x5d, 0x5a, 0xa2, 0xae, + 0x21, 0x61, 0xb2, 0xb4, 0xe2, 0x52, 0xda, 0xd4, 0xf8, 0x14, 0x0a, 0x11, 0xf9, 0x56, 0x6d, 0x19, + 0xbf, 0xa9, 0xb0, 0x1d, 0x4a, 0x1e, 0xdd, 0x52, 0xbc, 0x51, 0xe5, 0xee, 0x46, 0xd3, 0xc9, 0x46, + 0x3f, 0xe1, 0x14, 0x9b, 0x2f, 0x48, 0x40, 0x75, 0x55, 0xa4, 0xad, 0x24, 0xd2, 0x1e, 0x87, 0xa4, + 0xcc, 0xbe, 0xf6, 0x45, 0x2d, 0xd8, 0xe5, 0x21, 0x03, 0x42, 0xbd, 0xe5, 0x8a, 0xd9, 0x9e, 0x3b, + 0x3d, 0xb3, 0xdd, 0x53, 0xef, 0x4c, 0x88, 0xa5, 0xe2, 0x1d, 0xc7, 0x3c, 0xc7, 0x6b, 0x6e, 0x22, + 0x28, 0xf4, 0x01, 0x80, 0x69, 0x59, 0x01, 0xb1, 0x4c, 0x46, 0x42, 0x8d, 0xca, 0xad, 0xad, 0x28, + 0x5b, 0xdb, 0xb2, 0x02, 0x1c, 0xe3, 0xd1, 0xe7, 0xb0, 0xef, 0x9b, 0x01, 0xb3, 0xcd, 0x25, 0xcf, + 0x22, 0x6e, 0x7e, 0x7a, 0x6a, 0x53, 0x73, 0xb6, 0x24, 0xa7, 0x7a, 0xae, 0xae, 0x34, 0x0a, 0xf8, + 0xb1, 0x74, 0x88, 0x26, 0xe3, 0x99, 0xa4, 0xd1, 0x77, 0xb7, 0xfc, 0x96, 0xb2, 0xc0, 0x64, 0xc4, + 0xba, 0xd0, 0xf3, 0xe2, 0x3a, 0x0f, 0xa2, 0xc4, 0x5f, 0x26, 0x63, 0x0c, 0xa5, 0xdb, 0xff, 0x82, + 0x47, 0x04, 0x3a, 0x80, 0x12, 0x7d, 0x6d, 0xfb, 0xd3, 0xf9, 0x62, 0xe5, 0xbe, 0xa6, 0x7a, 0x41, + 0x94, 0x02, 0x1c, 0xea, 0x0a, 0xc4, 0xf8, 0x1e, 0xca, 0xd1, 0xd5, 0xc8, 0x89, 0x6d, 0x40, 0x6e, + 0xbd, 0x3d, 0x4a, 0xa3, 0xd4, 0x2a, 0xaf, 0x67, 0x49, 0xa0, 0x2f, 0x52, 0x58, 0xf2, 0xa8, 0x0a, + 0xf9, 0x33, 0x33, 0x70, 0x6d, 0xd7, 0x0a, 0xd7, 0xe5, 0x45, 0x0a, 0x47, 0x40, 0xa7, 0x00, 0xb9, + 0x80, 0xd0, 0xd5, 0x92, 0x19, 0xbf, 0x2b, 0xf0, 0x48, 0x5c, 0xcf, 0xc0, 0x74, 0x36, 0x13, 0x70, + 0xaf, 0x62, 0xca, 0x03, 0x14, 0x4b, 0x3f, 0x4c, 0x31, 0xe3, 0x39, 0xa0, 0x78, 0xb5, 0x52, 0x94, + 0x0a, 0x64, 0x5d, 0x0e, 0x88, 0x71, 0x2f, 0xe2, 0xd0, 0x40, 0x55, 0x28, 0xc8, 0x7e, 0xa9, 0x9e, + 0x16, 0xc4, 0xda, 0x36, 0xfe, 0x50, 0x64, 0xa0, 0x57, 0xe6, 0x72, 0xb5, 0xe9, 0xbb, 0x02, 0x59, + 0xb1, 0x15, 0xa2, 0xc7, 0x22, 0x0e, 0x8d, 0xfb, 0xd5, 0x48, 0x3f, 0x40, 0x0d, 0xf5, 0x81, 0x6a, + 0xf4, 0x61, 0x27, 0xd1, 0x84, 0x94, 0x63, 0x0f, 0x72, 0x3f, 0x09, 0x44, 0xea, 0x21, 0xad, 0xfb, + 0x04, 0x39, 0xc4, 0x50, 0x5c, 0x7f, 0x8d, 0x50, 0x09, 0xf2, 0xe3, 0xc1, 0x17, 0x83, 0x93, 0xc9, + 0x40, 0x4b, 0xa1, 0x22, 0x64, 0xbf, 0x1a, 0xf7, 0xf0, 0x37, 0x9a, 0x82, 0x0a, 0x90, 0xc1, 0xe3, + 0x97, 0x3d, 0x2d, 0xcd, 0x3d, 0x86, 0xfd, 0x67, 0xbd, 0x6e, 0x1b, 0x6b, 0x2a, 0xf7, 0x18, 0x8e, + 0x4e, 0x70, 0x4f, 0xcb, 0x70, 0x1c, 0xf7, 0xba, 0xbd, 0xfe, 0xab, 0x9e, 0x96, 0x3d, 0x6c, 0xc2, + 0xe3, 0x3b, 0x5a, 0xe2, 0x91, 0x26, 0x6d, 0x2c, 0xc3, 0xb7, 0x3b, 0x27, 0x78, 0xa4, 0x29, 0x87, + 0x1d, 0xc8, 0xf0, 0xdd, 0x45, 0x79, 0x50, 0x71, 0x7b, 0x12, 0x72, 0xdd, 0x93, 0xf1, 0x60, 0xa4, + 0x29, 0x1c, 0x1b, 0x8e, 0x8f, 0xb5, 0x34, 0x3f, 0x1c, 0xf7, 0x07, 0x9a, 0x2a, 0x0e, 0xed, 0xaf, + 0xc3, 0x9c, 0xc2, 0xab, 0x87, 0xb5, 0x6c, 0xeb, 0x97, 0x34, 0x64, 0x45, 0x23, 0xe8, 0x23, 0xc8, + 0xf0, 0x6f, 0x3d, 0xda, 0x89, 0xe4, 0x8d, 0xbd, 0x04, 0xd5, 0x4a, 0x12, 0x94, 0xc2, 0x7d, 0x06, + 0xb9, 0x70, 0x8d, 0xd0, 0x6e, 0x72, 0xad, 0xa2, 0x9f, 0xed, 0xdd, 0x84, 0xc3, 0x1f, 0x7e, 0xa8, + 0xa0, 0x2e, 0xc0, 0x66, 0x30, 0xd1, 0x7e, 0xe2, 0xcb, 0x17, 0x5f, 0xad, 0x6a, 0xf5, 0x36, 0x4a, + 0xe6, 0x7f, 0x0e, 0xa5, 0xd8, 0x7d, 0xa2, 0xa4, 0x6b, 0x62, 0x52, 0xab, 0xef, 0xde, 0xca, 0x85, + 0x71, 0x5a, 0x03, 0x28, 0x8b, 0x67, 0x97, 0x8f, 0x60, 0x28, 0xc6, 0x53, 0x28, 0x61, 0xe2, 0x78, + 0x8c, 0x08, 0x1c, 0xad, 0xdb, 0x8f, 0xbf, 0xce, 0xd5, 0xdd, 0x1b, 0xa8, 0x7c, 0xc4, 0x53, 0x9d, + 0xf7, 0x2e, 0xff, 0xad, 0xa5, 0x2e, 0xaf, 0x6a, 0xca, 0x9b, 0xab, 0x9a, 0xf2, 0xcf, 0x55, 0x4d, + 0xf9, 0xf5, 0xba, 0x96, 0x7a, 0x73, 0x5d, 0x4b, 0xfd, 0x79, 0x5d, 0x4b, 0x7d, 0x9b, 0x17, 0x6f, + 0x97, 0x3f, 0x9b, 0xe5, 0xc4, 0xbf, 0x80, 0x27, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x67, 0x56, + 0x73, 0x8e, 0x72, 0x08, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -849,6 +935,152 @@ var _Store_serviceDesc = grpc.ServiceDesc{ Metadata: "rpc.proto", } +// WriteableStoreClient is the client API for WriteableStore service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type WriteableStoreClient interface { + // WriteRequest allows you to write metrics to this store via remote write + RemoteWrite(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) +} + +type writeableStoreClient struct { + cc *grpc.ClientConn +} + +func NewWriteableStoreClient(cc *grpc.ClientConn) WriteableStoreClient { + return &writeableStoreClient{cc} +} + +func (c *writeableStoreClient) RemoteWrite(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) { + out := new(WriteResponse) + err := c.cc.Invoke(ctx, "/thanos.WriteableStore/RemoteWrite", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// WriteableStoreServer is the server API for WriteableStore service. +type WriteableStoreServer interface { + // WriteRequest allows you to write metrics to this store via remote write + RemoteWrite(context.Context, *WriteRequest) (*WriteResponse, error) +} + +// UnimplementedWriteableStoreServer can be embedded to have forward compatible implementations. +type UnimplementedWriteableStoreServer struct { +} + +func (*UnimplementedWriteableStoreServer) RemoteWrite(ctx context.Context, req *WriteRequest) (*WriteResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RemoteWrite not implemented") +} + +func RegisterWriteableStoreServer(s *grpc.Server, srv WriteableStoreServer) { + s.RegisterService(&_WriteableStore_serviceDesc, srv) +} + +func _WriteableStore_RemoteWrite_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(WriteRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WriteableStoreServer).RemoteWrite(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/thanos.WriteableStore/RemoteWrite", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WriteableStoreServer).RemoteWrite(ctx, req.(*WriteRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _WriteableStore_serviceDesc = grpc.ServiceDesc{ + ServiceName: "thanos.WriteableStore", + HandlerType: (*WriteableStoreServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "RemoteWrite", + Handler: _WriteableStore_RemoteWrite_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "rpc.proto", +} + +func (m *WriteResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *WriteResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *WriteResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + return len(dAtA) - i, nil +} + +func (m *WriteRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *WriteRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *WriteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Replica != 0 { + i = encodeVarintRpc(dAtA, i, uint64(m.Replica)) + i-- + dAtA[i] = 0x18 + } + if len(m.Tenant) > 0 { + i -= len(m.Tenant) + copy(dAtA[i:], m.Tenant) + i = encodeVarintRpc(dAtA, i, uint64(len(m.Tenant))) + i-- + dAtA[i] = 0x12 + } + if len(m.Timeseries) > 0 { + for iNdEx := len(m.Timeseries) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Timeseries[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRpc(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *InfoRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1313,6 +1545,37 @@ func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } +func (m *WriteResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + return n +} + +func (m *WriteRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Timeseries) > 0 { + for _, e := range m.Timeseries { + l = e.Size() + n += 1 + l + sovRpc(uint64(l)) + } + } + l = len(m.Tenant) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + if m.Replica != 0 { + n += 1 + sovRpc(uint64(m.Replica)) + } + return n +} + func (m *InfoRequest) Size() (n int) { if m == nil { return 0 @@ -1523,6 +1786,197 @@ func sovRpc(x uint64) (n int) { func sozRpc(x uint64) (n int) { return sovRpc(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } +func (m *WriteResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: WriteResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: WriteResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *WriteRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: WriteRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: WriteRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Timeseries", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRpc + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Timeseries = append(m.Timeseries, prompb.TimeSeries{}) + if err := m.Timeseries[len(m.Timeseries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Tenant", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRpc + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Tenant = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Replica", wireType) + } + m.Replica = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Replica |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *InfoRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/store/storepb/rpc.proto b/pkg/store/storepb/rpc.proto index 219262f207..ebd3bb20c7 100644 --- a/pkg/store/storepb/rpc.proto +++ b/pkg/store/storepb/rpc.proto @@ -3,6 +3,7 @@ package thanos; import "types.proto"; import "gogoproto/gogo.proto"; +import "github.com/prometheus/prometheus/prompb/types.proto"; option go_package = "storepb"; @@ -39,6 +40,21 @@ service Store { rpc LabelValues(LabelValuesRequest) returns (LabelValuesResponse); } +/// WriteableStore reprents API against instance that stores XOR encoded values with label set metadata (e.g Prometheus metrics). +service WriteableStore { + // WriteRequest allows you to write metrics to this store via remote write + rpc RemoteWrite(WriteRequest) returns (WriteResponse) {} +} + +message WriteResponse { +} + +message WriteRequest { + repeated prometheus.TimeSeries timeseries = 1 [(gogoproto.nullable) = false]; + string tenant = 2; + int64 replica = 3; +} + message InfoRequest { } @@ -108,11 +124,11 @@ enum Aggr { message SeriesResponse { oneof result { - Series series = 1; + Series series = 1; - /// warning is considered an information piece in place of series for warning purposes. - /// It is used to warn query customer about suspicious cases or partial response (if enabled). - string warning = 2; + /// warning is considered an information piece in place of series for warning purposes. + /// It is used to warn query customer about suspicious cases or partial response (if enabled). + string warning = 2; } } diff --git a/pkg/store/tsdb.go b/pkg/store/tsdb.go index 40de141811..3ea87412a6 100644 --- a/pkg/store/tsdb.go +++ b/pkg/store/tsdb.go @@ -28,6 +28,12 @@ type TSDBStore struct { externalLabels labels.Labels } +// ReadWriteTSDBStore is a TSDBStore that can also be written to. +type ReadWriteTSDBStore struct { + storepb.StoreServer + storepb.WriteableStoreServer +} + // NewTSDBStore creates a new TSDBStore. func NewTSDBStore(logger log.Logger, _ prometheus.Registerer, db *tsdb.DB, component component.SourceStoreAPI, externalLabels labels.Labels) *TSDBStore { if logger == nil { diff --git a/scripts/genproto.sh b/scripts/genproto.sh index f0e029f766..2df48ba70f 100755 --- a/scripts/genproto.sh +++ b/scripts/genproto.sh @@ -30,8 +30,9 @@ echo "generating code" for dir in ${DIRS}; do pushd ${dir} ${PROTOC_BIN} --gogofast_out=plugins=grpc:. -I=. \ - -I="${GOGOPROTO_PATH}" \ - *.proto + -I="${GOGOPROTO_PATH}" \ + -I="../../../vendor" \ + *.proto sed -i.bak -E 's/import _ \"gogoproto\"//g' *.pb.go sed -i.bak -E 's/import _ \"google\/protobuf\"//g' *.pb.go diff --git a/test/e2e/receive_test.go b/test/e2e/receive_test.go index ae700711b5..be2e1118ef 100644 --- a/test/e2e/receive_test.go +++ b/test/e2e/receive_test.go @@ -33,15 +33,15 @@ func TestReceive(t *testing.T) { // receiver in the hashring than the one handling the request. // The querier queries all the receivers and the test verifies // the time series are forwarded to the correct receive node. - receiveHTTP1, receiveHTTP2, receiveHTTP3 := a.New(), a.New(), a.New() + receiveGRPC1, receiveGRPC2, receiveGRPC3 := a.New(), a.New(), a.New() h := receive.HashringConfig{ - Endpoints: []string{remoteWriteEndpoint(receiveHTTP1), remoteWriteEndpoint(receiveHTTP2), remoteWriteEndpoint(receiveHTTP3)}, + Endpoints: []string{receiveGRPC1.HostPort(), receiveGRPC2.HostPort(), receiveGRPC3.HostPort()}, } - r1 := receiver(receiveHTTP1, a.New(), a.New(), 1, h) - r2 := receiver(receiveHTTP2, a.New(), a.New(), 1, h) - r3 := receiver(receiveHTTP3, a.New(), a.New(), 1, h) + r1 := receiver(a.New(), receiveGRPC1, a.New(), 1, h) + r2 := receiver(a.New(), receiveGRPC2, a.New(), 1, h) + r3 := receiver(a.New(), receiveGRPC3, a.New(), 1, h) prom1 := prometheus(a.New(), defaultPromConfig("prom1", 1, remoteWriteEndpoint(r1.HTTP))) prom2 := prometheus(a.New(), defaultPromConfig("prom2", 1, remoteWriteEndpoint(r2.HTTP))) @@ -79,15 +79,15 @@ func TestReceive(t *testing.T) { // receives Prometheus remote-written data. The querier queries all // receivers and the test verifies that the time series are // replicated to all of the nodes. - receiveHTTP1, receiveHTTP2, receiveHTTP3 := a.New(), a.New(), a.New() + receiveGRPC1, receiveGRPC2, receiveGRPC3 := a.New(), a.New(), a.New() h := receive.HashringConfig{ - Endpoints: []string{remoteWriteEndpoint(receiveHTTP1), remoteWriteEndpoint(receiveHTTP2), remoteWriteEndpoint(receiveHTTP3)}, + Endpoints: []string{receiveGRPC1.HostPort(), receiveGRPC2.HostPort(), receiveGRPC3.HostPort()}, } - r1 := receiver(receiveHTTP1, a.New(), a.New(), 3, h) - r2 := receiver(receiveHTTP2, a.New(), a.New(), 3, h) - r3 := receiver(receiveHTTP3, a.New(), a.New(), 3, h) + r1 := receiver(a.New(), receiveGRPC1, a.New(), 3, h) + r2 := receiver(a.New(), receiveGRPC2, a.New(), 3, h) + r3 := receiver(a.New(), receiveGRPC3, a.New(), 3, h) prom1 := prometheus(a.New(), defaultPromConfig("prom1", 1, remoteWriteEndpoint(r1.HTTP))) @@ -122,14 +122,14 @@ func TestReceive(t *testing.T) { // The replication suite creates a three-node hashring but one of the // receivers is dead. In this case, replication should still // succeed and the time series should be replicated to the other nodes. - receiveHTTP1, receiveHTTP2, receiveHTTP3 := a.New(), a.New(), a.New() + receiveGRPC1, receiveGRPC2, receiveGRPC3 := a.New(), a.New(), a.New() h := receive.HashringConfig{ - Endpoints: []string{remoteWriteEndpoint(receiveHTTP1), remoteWriteEndpoint(receiveHTTP2), remoteWriteEndpoint(receiveHTTP3)}, + Endpoints: []string{receiveGRPC1.HostPort(), receiveGRPC2.HostPort(), receiveGRPC3.HostPort()}, } - r1 := receiver(receiveHTTP1, a.New(), a.New(), 3, h) - r2 := receiver(receiveHTTP2, a.New(), a.New(), 3, h) + r1 := receiver(a.New(), receiveGRPC1, a.New(), 3, h) + r2 := receiver(a.New(), receiveGRPC2, a.New(), 3, h) prom1 := prometheus(a.New(), defaultPromConfig("prom1", 1, remoteWriteEndpoint(r1.HTTP))) diff --git a/test/e2e/spinup_test.go b/test/e2e/spinup_test.go index 54b34597ce..d51a51a60a 100644 --- a/test/e2e/spinup_test.go +++ b/test/e2e/spinup_test.go @@ -167,7 +167,7 @@ func sidecar(http, grpc address, prom *prometheusScheduler) *serverScheduler { func receiver(http, grpc, metric address, replicationFactor int, hashring ...receive.HashringConfig) *serverScheduler { if len(hashring) == 0 { - hashring = []receive.HashringConfig{{Endpoints: []string{remoteWriteEndpoint(http)}}} + hashring = []receive.HashringConfig{{Endpoints: []string{grpc.HostPort()}}} } return &serverScheduler{ @@ -198,7 +198,7 @@ func receiver(http, grpc, metric address, replicationFactor int, hashring ...rec "--tsdb.path", filepath.Join(receiveDir, "tsdb"), "--log.level", "debug", "--receive.replication-factor", strconv.Itoa(replicationFactor), - "--receive.local-endpoint", remoteWriteEndpoint(http), + "--receive.local-endpoint", grpc.HostPort(), "--receive.hashrings-file", filepath.Join(receiveDir, "hashrings.json"), "--receive.hashrings-file-refresh-interval", "5s")), nil }, From c45e52d32b66223c3ffa22bcad6ccc9d44023aaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Fri, 10 Jan 2020 19:29:00 +0200 Subject: [PATCH 167/257] Release 0.10.0-rc.1 (#1981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fixed crash in the memcached servers selector when there's only 1 server (#1975) * Fixed crash in the memcached servers selector when there's only 1 server Signed-off-by: Marco Pracucci * Updated changelog Signed-off-by: Marco Pracucci * Assert expected error in tests Signed-off-by: Marco Pracucci * *: update everything for 0.10.0-rc.1 Update the version strings before the `0.10.0-rc.1` release. Signed-off-by: Giedrius Statkevičius Co-authored-by: Marco Pracucci --- CHANGELOG.md | 3 +- VERSION | 2 +- pkg/cacheutil/memcached_server_selector.go | 5 +- .../memcached_server_selector_test.go | 48 +++++++++++++++++++ .../thanos/1-globalview/courseBase.sh | 2 +- .../katacoda/thanos/1-globalview/step2.md | 8 ++-- .../katacoda/thanos/1-globalview/step3.md | 2 +- 7 files changed, 61 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d8dc8e8dd..422cefdb47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.) -## [v0.10.0-rc.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0-rc.0) - 2020.01.08 +## [v0.10.0-rc.1](https://github.com/thanos-io/thanos/releases/tag/v0.10.0-rc.1) - 2020.01.10 ### Fixed @@ -29,6 +29,7 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1872](https://github.com/thanos-io/thanos/pull/1872) Ruler: `/api/v1/rules` now shows a properly formatted value - [#1945](https://github.com/thanos-io/thanos/pull/1945) `master` container images are now built with Go 1.13 - [#1956](https://github.com/thanos-io/thanos/pull/1956) Ruler: now properly ignores duplicated query addresses +- [#1975](https://github.com/thanos-io/thanos/pull/1975) Store Gateway: fixed panic caused by memcached servers selector when there's 1 memcached node ### Added diff --git a/VERSION b/VERSION index c8e421089d..2fe3f360c0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.10.0-rc.0 +0.10.0-rc.1 diff --git a/pkg/cacheutil/memcached_server_selector.go b/pkg/cacheutil/memcached_server_selector.go index 1318574045..37b693367c 100644 --- a/pkg/cacheutil/memcached_server_selector.go +++ b/pkg/cacheutil/memcached_server_selector.go @@ -74,9 +74,12 @@ func (s *MemcachedJumpHashSelector) PickServer(key string) (net.Addr, error) { return nil, memcache.ErrNoServers } if len(addrs) == 1 { + picked := addrs[0] + addrs = (addrs)[:0] addrsPool.Put(&addrs) - return (addrs)[0], nil + + return picked, nil } // Pick a server using the jump hash. diff --git a/pkg/cacheutil/memcached_server_selector_test.go b/pkg/cacheutil/memcached_server_selector_test.go index a215604ece..466db1c401 100644 --- a/pkg/cacheutil/memcached_server_selector_test.go +++ b/pkg/cacheutil/memcached_server_selector_test.go @@ -37,6 +37,54 @@ func TestNatSort(t *testing.T) { testutil.Equals(t, expected, input) } +func TestMemcachedJumpHashSelector_PickServer(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + tests := []struct { + addrs []string + key string + expectedAddr string + expectedErr error + }{ + { + addrs: []string{}, + key: "test-1", + expectedErr: memcache.ErrNoServers, + }, + { + addrs: []string{"127.0.0.1:11211"}, + key: "test-1", + expectedAddr: "127.0.0.1:11211", + }, + { + addrs: []string{"127.0.0.1:11211", "127.0.0.2:11211"}, + key: "test-1", + expectedAddr: "127.0.0.1:11211", + }, + { + addrs: []string{"127.0.0.1:11211", "127.0.0.2:11211"}, + key: "test-2", + expectedAddr: "127.0.0.2:11211", + }, + } + + s := MemcachedJumpHashSelector{} + + for _, test := range tests { + testutil.Ok(t, s.SetServers(test.addrs...)) + + actualAddr, err := s.PickServer(test.key) + + if test.expectedErr != nil { + testutil.Equals(t, test.expectedErr, err) + testutil.Equals(t, nil, actualAddr) + } else { + testutil.Ok(t, err) + testutil.Equals(t, test.expectedAddr, actualAddr.String()) + } + } +} + func TestMemcachedJumpHashSelector_Each_ShouldRespectServersOrdering(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() diff --git a/tutorials/katacoda/thanos/1-globalview/courseBase.sh b/tutorials/katacoda/thanos/1-globalview/courseBase.sh index dfa288e0d8..cad8757226 100644 --- a/tutorials/katacoda/thanos/1-globalview/courseBase.sh +++ b/tutorials/katacoda/thanos/1-globalview/courseBase.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash docker pull quay.io/prometheus/prometheus:v2.14.0 -docker pull quay.io/thanos/thanos:v0.10.0-rc.0 +docker pull quay.io/thanos/thanos:v0.10.0-rc.1 diff --git a/tutorials/katacoda/thanos/1-globalview/step2.md b/tutorials/katacoda/thanos/1-globalview/step2.md index 6cb687d3c1..569a9de690 100644 --- a/tutorials/katacoda/thanos/1-globalview/step2.md +++ b/tutorials/katacoda/thanos/1-globalview/step2.md @@ -10,7 +10,7 @@ component and can be invoked in a single command. Let's take a look at all the Thanos commands: ``` -docker run --rm quay.io/thanos/thanos:v0.10.0-rc.0 --help +docker run --rm quay.io/thanos/thanos:v0.10.0-rc.1 --help ```{{execute}} You should see multiple commands that solves different purposes. @@ -53,7 +53,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_eu1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-eu1 \ -u root \ - quay.io/thanos/thanos:v0.10.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0-rc.1 \ sidecar \ --http-address 0.0.0.0:19090 \ --grpc-address 0.0.0.0:19190 \ @@ -68,7 +68,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.10.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0-rc.1 \ sidecar \ --http-address 0.0.0.0:19091 \ --grpc-address 0.0.0.0:19191 \ @@ -81,7 +81,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus1_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-1-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.10.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0-rc.1 \ sidecar \ --http-address 0.0.0.0:19092 \ --grpc-address 0.0.0.0:19192 \ diff --git a/tutorials/katacoda/thanos/1-globalview/step3.md b/tutorials/katacoda/thanos/1-globalview/step3.md index bbdbb05981..987c918091 100644 --- a/tutorials/katacoda/thanos/1-globalview/step3.md +++ b/tutorials/katacoda/thanos/1-globalview/step3.md @@ -28,7 +28,7 @@ Click below snippet to start the Querier. ``` docker run -d --net=host --rm \ --name querier \ - quay.io/thanos/thanos:v0.10.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0-rc.1 \ query \ --http-address 0.0.0.0:29090 \ --query.replica-label replica \ From 4fc908e95195bc994789abc830cfd9fd5ebe5269 Mon Sep 17 00:00:00 2001 From: khyatisoneji <31898080+khyatisoneji@users.noreply.github.com> Date: Sun, 12 Jan 2020 17:23:03 +0530 Subject: [PATCH 168/257] bucket: use block.MetaFetcher to fetch blocks (#1988) Signed-off-by: khyatisoneji --- cmd/thanos/bucket.go | 118 ++++++++++++-------------- pkg/verifier/duplicated_compaction.go | 5 +- pkg/verifier/index_issue.go | 20 ++--- pkg/verifier/overlapped_blocks.go | 29 +++---- pkg/verifier/verify.go | 21 +++-- 5 files changed, 87 insertions(+), 106 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 157706ef3a..d49c360659 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -25,6 +25,7 @@ import ( "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/extprom" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/client" @@ -38,6 +39,8 @@ import ( kingpin "gopkg.in/alecthomas/kingpin.v2" ) +const extpromPrefix = "thanos_bucket_" + var ( issuesMap = map[string]verifier.Issue{ verifier.IndexIssueID: verifier.IndexIssue, @@ -123,10 +126,15 @@ func registerBucketVerify(m map[string]setupFunc, root *kingpin.CmdClause, name issues = append(issues, issueFn) } + fetcher, err := block.NewMetaFetcher(logger, fetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg)) + if err != nil { + return err + } + if *repair { - v = verifier.NewWithRepair(logger, bkt, backupBkt, issues) + v = verifier.NewWithRepair(logger, bkt, backupBkt, fetcher, issues) } else { - v = verifier.New(logger, bkt, issues) + v = verifier.New(logger, bkt, fetcher, issues) } var idMatcher func(ulid.ULID) bool = nil @@ -167,6 +175,11 @@ func registerBucketLs(m map[string]setupFunc, root *kingpin.CmdClause, name stri return err } + fetcher, err := block.NewMetaFetcher(logger, fetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg)) + if err != nil { + return err + } + // Dummy actor to immediately kill the group after the run function returns. g.Add(func() error { return nil }, func(error) {}) @@ -178,22 +191,17 @@ func registerBucketLs(m map[string]setupFunc, root *kingpin.CmdClause, name stri var ( format = *output objects = 0 - printBlock func(id ulid.ULID) error + printBlock func(m *metadata.Meta) error ) switch format { case "": - printBlock = func(id ulid.ULID) error { - fmt.Fprintln(os.Stdout, id.String()) + printBlock = func(m *metadata.Meta) error { + fmt.Fprintln(os.Stdout, m.ULID.String()) return nil } case "wide": - printBlock = func(id ulid.ULID) error { - m, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - return err - } - + printBlock = func(m *metadata.Meta) error { minTime := time.Unix(m.MinTime/1000, 0) maxTime := time.Unix(m.MaxTime/1000, 0) @@ -208,11 +216,7 @@ func registerBucketLs(m map[string]setupFunc, root *kingpin.CmdClause, name stri enc := json.NewEncoder(os.Stdout) enc.SetIndent("", "\t") - printBlock = func(id ulid.ULID) error { - m, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - return err - } + printBlock = func(m *metadata.Meta) error { return enc.Encode(&m) } default: @@ -220,12 +224,7 @@ func registerBucketLs(m map[string]setupFunc, root *kingpin.CmdClause, name stri if err != nil { return errors.Wrap(err, "invalid template") } - printBlock = func(id ulid.ULID) error { - m, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - return err - } - + printBlock = func(m *metadata.Meta) error { if err := tmpl.Execute(os.Stdout, &m); err != nil { return errors.Wrap(err, "execute template") } @@ -234,15 +233,16 @@ func registerBucketLs(m map[string]setupFunc, root *kingpin.CmdClause, name stri } } - if err := bkt.Iter(ctx, "", func(name string) error { - id, ok := block.IsBlockDir(name) - if !ok { - return nil - } + metas, _, err := fetcher.Fetch(ctx) + if err != nil { + return err + } + + for _, meta := range metas { objects++ - return printBlock(id) - }); err != nil { - return errors.Wrap(err, "iter") + if err := printBlock(meta); err != nil { + return errors.Wrap(err, "iter") + } } level.Info(logger).Log("msg", "ls done", "objects", objects) return nil @@ -275,6 +275,11 @@ func registerBucketInspect(m map[string]setupFunc, root *kingpin.CmdClause, name return err } + fetcher, err := block.NewMetaFetcher(logger, fetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg)) + if err != nil { + return err + } + // Dummy actor to immediately kill the group after the run function returns. g.Add(func() error { return nil }, func(error) {}) @@ -284,25 +289,16 @@ func registerBucketInspect(m map[string]setupFunc, root *kingpin.CmdClause, name defer cancel() // Getting Metas. - var blockMetas []*metadata.Meta - if err = bkt.Iter(ctx, "", func(name string) error { - id, ok := block.IsBlockDir(name) - if !ok { - return nil - } - - m, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - return err - } - - blockMetas = append(blockMetas, &m) - - return nil - }); err != nil { + metas, _, err := fetcher.Fetch(ctx) + if err != nil { return err } + blockMetas := make([]*metadata.Meta, 0, len(metas)) + for _, meta := range metas { + blockMetas = append(blockMetas, meta) + } + return printTable(blockMetas, selectorLabels, *sortBy) } } @@ -383,13 +379,18 @@ func refresh(ctx context.Context, logger log.Logger, bucketUI *ui.Bucket, durati return errors.Wrap(err, "bucket client") } + fetcher, err := block.NewMetaFetcher(logger, fetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg)) + if err != nil { + return err + } + defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") return runutil.Repeat(duration, ctx.Done(), func() error { return runutil.RetryWithLog(logger, time.Minute, ctx.Done(), func() error { iterCtx, iterCancel := context.WithTimeout(ctx, timeout) defer iterCancel() - blocks, err := download(iterCtx, logger, bkt) + blocks, err := download(iterCtx, logger, bkt, fetcher) if err != nil { bucketUI.Set("[]", err) return err @@ -406,25 +407,16 @@ func refresh(ctx context.Context, logger log.Logger, bucketUI *ui.Bucket, durati }) } -func download(ctx context.Context, logger log.Logger, bkt objstore.Bucket) (blocks []metadata.Meta, err error) { +func download(ctx context.Context, logger log.Logger, bkt objstore.Bucket, fetcher *block.MetaFetcher) (blocks []metadata.Meta, err error) { level.Info(logger).Log("msg", "synchronizing block metadata") - if err = bkt.Iter(ctx, "", func(name string) error { - id, ok := block.IsBlockDir(name) - if !ok { - return nil - } - - meta, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - return err - } + metas, _, err := fetcher.Fetch(ctx) + if err != nil { + return nil, err + } - blocks = append(blocks, meta) - return nil - }); err != nil { - level.Error(logger).Log("err", err, "msg", "Failed to downloaded block metadata") - return blocks, err + for _, meta := range metas { + blocks = append(blocks, *meta) } level.Info(logger).Log("msg", "downloaded blocks meta.json", "num", len(blocks)) diff --git a/pkg/verifier/duplicated_compaction.go b/pkg/verifier/duplicated_compaction.go index e2e5f2a561..0909e6e8f9 100644 --- a/pkg/verifier/duplicated_compaction.go +++ b/pkg/verifier/duplicated_compaction.go @@ -13,6 +13,7 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/prometheus/tsdb" + "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/objstore" ) @@ -23,14 +24,14 @@ const DuplicatedCompactionIssueID = "duplicated_compaction" // until sync-delay passes. // The expected print of this are same overlapped blocks with exactly the same sources, time ranges and stats. // If repair is enabled, all but one duplicates are safely deleted. -func DuplicatedCompactionIssue(ctx context.Context, logger log.Logger, bkt objstore.Bucket, backupBkt objstore.Bucket, repair bool, idMatcher func(ulid.ULID) bool) error { +func DuplicatedCompactionIssue(ctx context.Context, logger log.Logger, bkt objstore.Bucket, backupBkt objstore.Bucket, repair bool, idMatcher func(ulid.ULID) bool, fetcher *block.MetaFetcher) error { if idMatcher != nil { return errors.Errorf("id matching is not supported by issue %s verifier", DuplicatedCompactionIssueID) } level.Info(logger).Log("msg", "started verifying issue", "with-repair", repair, "issue", DuplicatedCompactionIssueID) - overlaps, err := fetchOverlaps(ctx, logger, bkt) + overlaps, err := fetchOverlaps(ctx, logger, bkt, fetcher) if err != nil { return errors.Wrap(err, DuplicatedCompactionIssueID) } diff --git a/pkg/verifier/index_issue.go b/pkg/verifier/index_issue.go index a25e8e181e..6f31b63c17 100644 --- a/pkg/verifier/index_issue.go +++ b/pkg/verifier/index_issue.go @@ -25,15 +25,15 @@ const IndexIssueID = "index_issue" // If the replacement was created successfully it is uploaded to the bucket and the input // block is deleted. // NOTE: This also verifies all indexes against chunks mismatches and duplicates. -func IndexIssue(ctx context.Context, logger log.Logger, bkt objstore.Bucket, backupBkt objstore.Bucket, repair bool, idMatcher func(ulid.ULID) bool) error { +func IndexIssue(ctx context.Context, logger log.Logger, bkt objstore.Bucket, backupBkt objstore.Bucket, repair bool, idMatcher func(ulid.ULID) bool, fetcher *block.MetaFetcher) error { level.Info(logger).Log("msg", "started verifying issue", "with-repair", repair, "issue", IndexIssueID) - err := bkt.Iter(ctx, "", func(name string) error { - id, ok := block.IsBlockDir(name) - if !ok { - return nil - } + metas, _, err := fetcher.Fetch(ctx) + if err != nil { + return err + } + for id, meta := range metas { if idMatcher != nil && !idMatcher(id) { return nil } @@ -52,11 +52,6 @@ func IndexIssue(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bac return errors.Wrapf(err, "download index file %s", path.Join(id.String(), block.IndexFilename)) } - meta, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - return errors.Wrapf(err, "download meta file %s", id) - } - stats, err := block.GatherIndexIssueStats(logger, filepath.Join(tmpdir, block.IndexFilename), meta.MinTime, meta.MaxTime) if err != nil { return errors.Wrapf(err, "gather index issues %s", id) @@ -122,9 +117,6 @@ func IndexIssue(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bac } level.Info(logger).Log("msg", "all good, continuing", "id", id, "issue", IndexIssueID) return nil - }) - if err != nil { - return errors.Wrapf(err, "verify iter, issue %s", IndexIssueID) } level.Info(logger).Log("msg", "verified issue", "with-repair", repair, "issue", IndexIssueID) diff --git a/pkg/verifier/overlapped_blocks.go b/pkg/verifier/overlapped_blocks.go index 0e7be08e64..4c35328e41 100644 --- a/pkg/verifier/overlapped_blocks.go +++ b/pkg/verifier/overlapped_blocks.go @@ -18,14 +18,14 @@ const OverlappedBlocksIssueID = "overlapped_blocks" // OverlappedBlocksIssue checks bucket for blocks with overlapped time ranges. // No repair is available for this issue. -func OverlappedBlocksIssue(ctx context.Context, logger log.Logger, bkt objstore.Bucket, _ objstore.Bucket, repair bool, idMatcher func(ulid.ULID) bool) error { +func OverlappedBlocksIssue(ctx context.Context, logger log.Logger, bkt objstore.Bucket, _ objstore.Bucket, repair bool, idMatcher func(ulid.ULID) bool, fetcher *block.MetaFetcher) error { if idMatcher != nil { return errors.Errorf("id matching is not supported by issue %s verifier", OverlappedBlocksIssueID) } level.Info(logger).Log("msg", "started verifying issue", "with-repair", repair, "issue", OverlappedBlocksIssueID) - overlaps, err := fetchOverlaps(ctx, logger, bkt) + overlaps, err := fetchOverlaps(ctx, logger, bkt, fetcher) if err != nil { return errors.Wrap(err, OverlappedBlocksIssueID) } @@ -45,28 +45,19 @@ func OverlappedBlocksIssue(ctx context.Context, logger log.Logger, bkt objstore. return nil } -func fetchOverlaps(ctx context.Context, logger log.Logger, bkt objstore.Bucket) (map[string]tsdb.Overlaps, error) { - metas := map[string][]tsdb.BlockMeta{} - err := bkt.Iter(ctx, "", func(name string) error { - id, ok := block.IsBlockDir(name) - if !ok { - return nil - } - - m, err := block.DownloadMeta(ctx, logger, bkt, id) - if err != nil { - return err - } - - metas[compact.GroupKey(m.Thanos)] = append(metas[compact.GroupKey(m.Thanos)], m.BlockMeta) - return nil - }) +func fetchOverlaps(ctx context.Context, logger log.Logger, bkt objstore.Bucket, fetcher *block.MetaFetcher) (map[string]tsdb.Overlaps, error) { + metas, _, err := fetcher.Fetch(ctx) if err != nil { return nil, err } + groupMetasMap := map[string][]tsdb.BlockMeta{} + for _, meta := range metas { + groupMetasMap[compact.GroupKey(meta.Thanos)] = append(groupMetasMap[compact.GroupKey(meta.Thanos)], meta.BlockMeta) + } + overlaps := map[string]tsdb.Overlaps{} - for k, groupMetas := range metas { + for k, groupMetas := range groupMetasMap { sort.Slice(groupMetas, func(i, j int) bool { return groupMetas[i].MinTime < groupMetas[j].MinTime diff --git a/pkg/verifier/verify.go b/pkg/verifier/verify.go index 048bf8a588..1bddfe6d98 100644 --- a/pkg/verifier/verify.go +++ b/pkg/verifier/verify.go @@ -3,6 +3,8 @@ package verifier import ( "context" + "github.com/thanos-io/thanos/pkg/block" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/ulid" @@ -12,7 +14,7 @@ import ( // Issue is an function that does verification and repair only if repair arg is true. // It should log affected blocks using warn level logs. It should be safe for issue to run on healthy bucket. -type Issue func(ctx context.Context, logger log.Logger, bkt objstore.Bucket, backupBkt objstore.Bucket, repair bool, idMatcher func(ulid.ULID) bool) error +type Issue func(ctx context.Context, logger log.Logger, bkt objstore.Bucket, backupBkt objstore.Bucket, repair bool, idMatcher func(ulid.ULID) bool, fetcher *block.MetaFetcher) error // Verifier runs given issues to verify if bucket is healthy. type Verifier struct { @@ -21,25 +23,28 @@ type Verifier struct { backupBkt objstore.Bucket issues []Issue repair bool + fetcher *block.MetaFetcher } // New returns verifier that only logs affected blocks. -func New(logger log.Logger, bkt objstore.Bucket, issues []Issue) *Verifier { +func New(logger log.Logger, bkt objstore.Bucket, fetcher *block.MetaFetcher, issues []Issue) *Verifier { return &Verifier{ - logger: logger, - bkt: bkt, - issues: issues, - repair: false, + logger: logger, + bkt: bkt, + issues: issues, + fetcher: fetcher, + repair: false, } } // NewWithRepair returns verifier that logs affected blocks and attempts to repair them. -func NewWithRepair(logger log.Logger, bkt objstore.Bucket, backupBkt objstore.Bucket, issues []Issue) *Verifier { +func NewWithRepair(logger log.Logger, bkt objstore.Bucket, backupBkt objstore.Bucket, fetcher *block.MetaFetcher, issues []Issue) *Verifier { return &Verifier{ logger: logger, bkt: bkt, backupBkt: backupBkt, issues: issues, + fetcher: fetcher, repair: true, } } @@ -59,7 +64,7 @@ func (v *Verifier) Verify(ctx context.Context, idMatcher func(ulid.ULID) bool) e // TODO(blotka): Wrap bucket with BucketWithMetrics and print metrics after each issue (e.g how many blocks where touched). // TODO(bplotka): Implement disk "bucket" to allow this verify to work on local disk space as well. for _, issueFn := range v.issues { - err := issueFn(ctx, v.logger, v.bkt, v.backupBkt, v.repair, idMatcher) + err := issueFn(ctx, v.logger, v.bkt, v.backupBkt, v.repair, idMatcher, v.fetcher) if err != nil { return errors.Wrap(err, "verify") } From bce3109fd29adcfc0a70f164e4b8740425f44dbe Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Sun, 12 Jan 2020 12:59:07 +0100 Subject: [PATCH 169/257] store: Refetch series if longer than expected. (#1985) Fixes: https://github.com/thanos-io/thanos/issues/1983 Signed-off-by: Bartlomiej Plotka --- CHANGELOG.md | 1 + pkg/objstore/inmem/inmem.go | 11 ++++-- pkg/store/bucket.go | 47 ++++++++++++++++------- pkg/store/bucket_test.go | 75 +++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 985ba42fa4..657e7f2826 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Fixed +- [1985](https://github.com/thanos-io/thanos/pull/1985) store gateway: Fixed case where series entry is larger than 64KB in index. - [#1919](https://github.com/thanos-io/thanos/issues/1919) Compactor: Fixed potential data loss when uploading older blocks, or upload taking long time while compactor is running. - [#1937](https://github.com/thanos-io/thanos/pull/1937) Compactor: Improved synchronization of meta JSON files. diff --git a/pkg/objstore/inmem/inmem.go b/pkg/objstore/inmem/inmem.go index 8b04974272..8dd391d570 100644 --- a/pkg/objstore/inmem/inmem.go +++ b/pkg/objstore/inmem/inmem.go @@ -1,14 +1,13 @@ package inmem import ( + "bytes" "context" "io" - "sort" - "sync" - - "bytes" "io/ioutil" + "sort" "strings" + "sync" "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/objstore" @@ -123,6 +122,10 @@ func (b *Bucket) GetRange(_ context.Context, name string, off, length int64) (io return ioutil.NopCloser(bytes.NewReader(file[off:])), nil } + if length <= 0 { + return ioutil.NopCloser(bytes.NewReader(nil)), errors.New("length cannot be smaller or equal 0") + } + if int64(len(file)) <= off+length { // Just return maximum of what we have. length = int64(len(file)) - off diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 835147a32d..d4a711e192 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -57,6 +57,8 @@ const ( maxChunkSize = 16000 + maxSeriesSize = 64 * 1024 + // CompatibilityTypeLabelName is an artificial label that Store Gateway can optionally advertise. This is required for compatibility // with pre v0.8.0 Querier. Previous Queriers was strict about duplicated external labels of all StoreAPIs that had any labels. // Now with newer Store Gateway advertising all the external labels it has access to, there was simple case where @@ -87,6 +89,7 @@ type bucketStoreMetrics struct { chunkSizeBytes prometheus.Histogram queriesDropped prometheus.Counter queriesLimit prometheus.Gauge + seriesRefetches prometheus.Counter } func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { @@ -166,6 +169,10 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { Name: "thanos_bucket_store_queries_concurrent_max", Help: "Number of maximum concurrent queries.", }) + m.seriesRefetches = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "thanos_bucket_store_series_refetches_total", + Help: fmt.Sprintf("Total number of cases where %v bytes was not enough was to fetch series from index, resulting in refetch.", maxSeriesSize), + }) if reg != nil { reg.MustRegister( @@ -185,6 +192,7 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { m.chunkSizeBytes, m.queriesDropped, m.queriesLimit, + m.seriesRefetches, ) } return &m @@ -454,6 +462,7 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er s.chunkPool, jr, s.partitioner, + s.metrics.seriesRefetches, ) if err != nil { return errors.Wrap(err, "new bucket block") @@ -601,8 +610,6 @@ func (s *bucketSeriesSet) Err() error { } func blockSeries( - ctx context.Context, - ulid ulid.ULID, extLset map[string]string, indexr *bucketIndexReader, chunkr *bucketChunkReader, @@ -845,8 +852,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie defer runutil.CloseWithLogOnErr(s.logger, chunkr, "series block") g.Go(func() error { - part, pstats, err := blockSeries(ctx, - b.meta.ULID, + part, pstats, err := blockSeries( b.meta.Thanos.Labels, indexr, chunkr, @@ -1156,6 +1162,8 @@ type bucketBlock struct { pendingReaders sync.WaitGroup partitioner partitioner + + seriesRefetches prometheus.Counter } func newBucketBlock( @@ -1168,6 +1176,7 @@ func newBucketBlock( chunkPool *pool.BytesPool, indexHeadReader indexheader.Reader, p partitioner, + seriesRefetches prometheus.Counter, ) (b *bucketBlock, err error) { b = &bucketBlock{ logger: logger, @@ -1178,6 +1187,7 @@ func newBucketBlock( partitioner: p, meta: meta, indexHeaderReader: indexHeadReader, + seriesRefetches: seriesRefetches, } // Get object handles for all chunk files. @@ -1484,12 +1494,11 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { fetchTime := time.Since(begin) r.mtx.Lock() - defer r.mtx.Unlock() - r.stats.postingsFetchCount++ r.stats.postingsFetched += j - i r.stats.postingsFetchDurationSum += fetchTime r.stats.postingsFetchedSizeSum += int(length) + r.mtx.Unlock() for _, p := range ptrs[i:j] { c := b[p.ptr.Start-start : p.ptr.End-start] @@ -1499,6 +1508,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { return errors.Wrap(err, "read postings list") } + r.mtx.Lock() // Return postings and fill LRU cache. groups[p.groupID].Fill(p.keyID, fetchedPostings) r.cache.StorePostings(r.ctx, r.block.meta.ULID, groups[p.groupID].keys[p.keyID], c) @@ -1506,6 +1516,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { // If we just fetched it we still have to update the stats for touched postings. r.stats.postingsTouched++ r.stats.postingsTouchedSizeSum += len(c) + r.mtx.Unlock() } return nil }) @@ -1515,8 +1526,6 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { } func (r *bucketIndexReader) PreloadSeries(ids []uint64) error { - const maxSeriesSize = 64 * 1024 - // Load series from cache, overwriting the list of ids to preload // with the missing ones. fromCache, ids := r.cache.FetchMultiSeries(r.ctx, r.block.meta.ULID, ids) @@ -1533,13 +1542,13 @@ func (r *bucketIndexReader) PreloadSeries(ids []uint64) error { i, j := p.elemRng[0], p.elemRng[1] g.Go(func() error { - return r.loadSeries(ctx, ids[i:j], s, e) + return r.loadSeries(ctx, ids[i:j], false, s, e) }) } return g.Wait() } -func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, start, end uint64) error { +func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, refetch bool, start, end uint64) error { begin := time.Now() b, err := r.block.readIndexRange(ctx, int64(start), int64(end-start)) @@ -1548,14 +1557,13 @@ func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, start, } r.mtx.Lock() - defer r.mtx.Unlock() - r.stats.seriesFetchCount++ r.stats.seriesFetched += len(ids) r.stats.seriesFetchDurationSum += time.Since(begin) r.stats.seriesFetchedSizeSum += int(end - start) + r.mtx.Unlock() - for _, id := range ids { + for i, id := range ids { c := b[id-start:] l, n := binary.Uvarint(c) @@ -1563,11 +1571,22 @@ func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, start, return errors.New("reading series length failed") } if len(c) < n+int(l) { - return errors.Errorf("invalid remaining size %d, expected %d", len(c), n+int(l)) + if i == 0 && refetch { + return errors.Errorf("invalid remaining size, even after refetch, remaining: %d, expected %d", len(c), n+int(l)) + } + + // Inefficient, but should be rare. + r.block.seriesRefetches.Inc() + level.Warn(r.logger).Log("msg", "series size exceeded expected size; refetching", "id", id, "series length", n+int(l), "maxSeriesSize", maxSeriesSize) + + // Fetch plus to get the size of next one if exists. + return r.loadSeries(ctx, ids[i:], true, id, id+uint64(n+int(l)+1)) } c = c[n : n+int(l)] + r.mtx.Lock() r.loadedSeries[id] = c r.cache.StoreSeries(r.ctx, r.block.meta.ULID, id, c) + r.mtx.Unlock() } return nil } diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 913714f655..cb861347a2 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "context" "fmt" "io" @@ -20,8 +21,11 @@ import ( "github.com/leanovate/gopter/gen" "github.com/leanovate/gopter/prop" "github.com/oklog/ulid" + promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" + "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/encoding" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact/downsample" @@ -760,3 +764,74 @@ func expectedTouchedBlockOps(all []ulid.ULID, expected []ulid.ULID, cached []uli sort.Strings(ops) return ops } + +// Regression tests against: https://github.com/thanos-io/thanos/issues/1983. +func TestReadIndexCache_LoadSeries(t *testing.T) { + bkt := inmem.NewBucket() + + s := newBucketStoreMetrics(nil) + b := &bucketBlock{ + meta: &metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil)}, + }, + bucket: bkt, + seriesRefetches: s.seriesRefetches, + logger: log.NewNopLogger(), + } + + buf := encoding.Encbuf{} + buf.PutByte(0) + buf.PutByte(0) + buf.PutUvarint(10) + buf.PutString("aaaaaaaaaa") + buf.PutUvarint(10) + buf.PutString("bbbbbbbbbb") + buf.PutUvarint(10) + buf.PutString("cccccccccc") + testutil.Ok(t, bkt.Upload(context.Background(), filepath.Join(b.meta.ULID.String(), block.IndexFilename), bytes.NewReader(buf.Get()))) + + r := bucketIndexReader{ + block: b, + stats: &queryStats{}, + loadedSeries: map[uint64][]byte{}, + cache: noopCache{}, + logger: log.NewNopLogger(), + } + + // Success with no refetches. + testutil.Ok(t, r.loadSeries(context.TODO(), []uint64{2, 13, 24}, false, 2, 100)) + testutil.Equals(t, map[uint64][]byte{ + 2: []byte("aaaaaaaaaa"), + 13: []byte("bbbbbbbbbb"), + 24: []byte("cccccccccc"), + }, r.loadedSeries) + testutil.Equals(t, float64(0), promtest.ToFloat64(s.seriesRefetches)) + + // Success with 2 refetches. + r.loadedSeries = map[uint64][]byte{} + testutil.Ok(t, r.loadSeries(context.TODO(), []uint64{2, 13, 24}, false, 2, 15)) + testutil.Equals(t, map[uint64][]byte{ + 2: []byte("aaaaaaaaaa"), + 13: []byte("bbbbbbbbbb"), + 24: []byte("cccccccccc"), + }, r.loadedSeries) + testutil.Equals(t, float64(2), promtest.ToFloat64(s.seriesRefetches)) + + // Success with refetch on first element. + r.loadedSeries = map[uint64][]byte{} + testutil.Ok(t, r.loadSeries(context.TODO(), []uint64{2}, false, 2, 5)) + testutil.Equals(t, map[uint64][]byte{ + 2: []byte("aaaaaaaaaa"), + }, r.loadedSeries) + testutil.Equals(t, float64(3), promtest.ToFloat64(s.seriesRefetches)) + + buf.Reset() + buf.PutByte(0) + buf.PutByte(0) + buf.PutUvarint(10) + buf.PutString("aaaaaaa") + testutil.Ok(t, bkt.Upload(context.Background(), filepath.Join(b.meta.ULID.String(), block.IndexFilename), bytes.NewReader(buf.Get()))) + + // Fail, but no recursion at least. + testutil.NotOk(t, r.loadSeries(context.TODO(), []uint64{2, 13, 24}, false, 1, 15)) +} From 46e5f45044d49c6440aca93f1b81b8163c974501 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 12 Jan 2020 14:21:47 -0500 Subject: [PATCH 170/257] persist option in query UI (#1870) Signed-off-by: yeya24 --- pkg/ui/bindata.go | 12 +++--- pkg/ui/static/js/graph.js | 68 ++++++++++++++++++++------------ pkg/ui/templates/query_menu.html | 40 +++++++++---------- 3 files changed, 68 insertions(+), 52 deletions(-) diff --git a/pkg/ui/bindata.go b/pkg/ui/bindata.go index d9cd32f615..d0d47c5c5f 100644 --- a/pkg/ui/bindata.go +++ b/pkg/ui/bindata.go @@ -220,7 +220,7 @@ func pkgUiTemplatesBucket_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 787, mode: os.FileMode(420), modTime: time.Unix(1575985294, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 787, mode: os.FileMode(420), modTime: time.Unix(1576072884, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -245,7 +245,7 @@ func pkgUiTemplatesGraphHtml() (*asset, error) { return a, nil } -var _pkgUiTemplatesQuery_menuHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x54\x31\x8f\x9c\x3c\x10\xad\xbf\xfb\x15\x23\x9f\xf4\x75\x9c\xfb\x04\x28\xd2\x24\xe9\xa2\xdc\xf5\xd1\xb0\x1e\xc0\x5a\x63\x5b\xf6\xb0\xd9\x08\xf1\xdf\x23\xb3\xbb\xc4\xb0\xb7\x52\x8a\x54\xc8\xa3\xe7\x37\xef\xcd\x3c\x33\x4d\x8a\x5a\x6d\x09\x84\xc5\x93\x98\xe7\x27\x00\x80\xd2\xe2\x09\x0e\x06\x63\xac\x52\xb9\xc1\x00\xad\x3e\x93\x2a\xd8\x79\xb8\x14\x0a\x3a\x7b\xb4\xaa\x88\xc3\xad\xa0\x30\x1c\xa1\xe9\x96\xaf\xa8\x17\x1e\x80\x52\xe9\x95\xe9\xe0\x2c\xa3\xb6\x14\x8a\xd6\x8c\x5a\xad\x18\x80\xb2\x19\x99\x9d\x05\xfe\xe5\xa9\x12\x97\x83\xd8\x0a\x28\xd8\x75\x9d\xa1\x20\x40\x21\xe3\xf5\x94\x38\x8d\x41\x1f\xe9\x56\xc6\xd0\x11\x57\xe2\xd9\xe2\xa9\x48\xfd\xc8\xb2\x00\x0c\x1a\xaf\x7a\x49\x55\xa2\x45\x93\x2e\x2c\xd5\x84\x09\xce\x5c\xda\xec\x6e\x18\x6c\xc8\x54\xe2\x6d\x69\x95\x5c\xea\x0e\x59\x3b\x9b\x09\x07\x28\xa3\x47\xfb\xbe\xd4\x42\x1f\x12\xb8\x94\x09\x92\x99\x95\x17\x83\x59\x05\x77\x04\x4d\x40\xab\x04\xf4\x81\xda\x4a\x4c\x13\x78\xe4\xfe\x5b\xa0\x56\x9f\x61\x9e\xa5\xa8\xdf\x7a\xb4\x2e\x96\x12\x33\x8e\x34\x68\xad\x76\x3e\xb6\xb4\xb7\x61\xc1\x3a\xb5\x8d\x93\xd1\xec\xf0\x29\x11\x39\x02\xa0\x34\x3a\xc3\x14\x9a\x69\x10\xf5\x46\x7e\x61\xb4\x3d\x3e\x94\xde\x05\xf4\xbd\xa8\x3f\xa7\x4f\x92\x5f\x4a\xa3\xff\x6d\x87\xc8\x2e\x50\x14\xf5\xeb\xf2\xfd\xd3\xe3\xbf\x07\xe4\xa0\x82\xf3\xca\xfd\xb4\x3b\xa7\xcb\x56\x2e\x3d\x9e\xc5\xbe\xfb\x7a\xe9\xba\xea\x5d\x28\x57\x4a\x08\xce\x64\x81\x5e\x52\xd5\x63\xf4\xce\x8f\xbe\x12\x1c\x46\x7a\x10\xce\xfa\x95\x91\xc7\xb8\x4d\xd7\x01\x03\xf1\x9a\xa7\xcd\xf6\xef\x1f\xdb\xaa\x70\x20\x3b\xde\x79\xcb\x33\xb7\x22\x97\x59\x3f\x9e\x6b\x12\x24\xea\xef\xa3\x65\x3d\x10\xfc\x8f\x83\xff\x08\x9f\x46\x6d\x14\x7c\xb5\xad\x0b\xc3\xf2\x36\xde\x53\x25\x95\x3e\xed\x96\xfc\x77\x6b\xbf\x5f\xc8\x83\x14\xf4\xcc\x3e\x7e\x90\x92\x97\x77\xf1\xa2\x9d\xec\x88\x59\xdb\xae\x88\x8c\x81\x49\xbd\x0c\x4a\x0a\xb8\xfd\x1d\x7e\x34\x06\xed\x51\xd4\x5f\xc8\xf8\x3b\xc1\x7b\x6d\xa5\x1c\x4d\xfe\x78\x33\x33\xd9\xa1\x94\x16\x4f\xf5\xd3\x34\x91\x55\xf3\xfc\xf4\x3b\x00\x00\xff\xff\xe6\xe9\x64\x17\x56\x05\x00\x00") +var _pkgUiTemplatesQuery_menuHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x54\xb1\x6e\xdc\x30\x0c\xdd\xf3\x15\x84\x02\x74\x73\xb4\xb7\xb6\x87\x2e\x6d\xb7\xa2\xc9\x5e\xd0\x27\xda\x16\x22\x4b\x82\x44\x5f\x53\x18\xfe\xf7\x42\xba\x3b\xe3\xec\x9c\x0f\xc8\xd0\x49\x10\x41\x3e\xbe\x27\x3e\x6a\x9a\x14\xb5\xda\x12\x08\x8b\x47\x31\xcf\x0f\x00\x00\xa5\xc5\x23\x1c\x0c\xc6\x58\xa5\x70\x83\x01\x5a\xfd\x46\xaa\x60\xe7\xe1\x14\x28\xe8\xcd\xa3\x55\x45\x1c\x2e\x01\x85\xe1\x15\x9a\x2e\x9f\xa2\xce\x38\x19\x4b\xe9\x05\xeb\xe0\x2c\xa3\xb6\x14\x8a\xd6\x8c\x5a\x5d\x65\xe5\xcc\x66\x64\x76\x16\xf8\xaf\xa7\x4a\x9c\x2e\x62\x4d\xa3\x60\xd7\x75\x86\x82\x00\x85\x8c\xe7\x5b\xc2\x35\x06\x7d\xa4\x4b\x18\x43\x47\x5c\x89\x47\x8b\xc7\x22\xf5\x24\xcb\x02\x30\x68\x3c\xb3\x26\x55\x89\x16\x4d\x2a\xc8\xd1\x94\x13\x9c\x39\xb5\xd9\x54\x18\x6c\xc8\x54\xe2\x25\xb7\x4a\x5a\x75\x87\xac\x9d\xdd\x90\xcf\x02\xa2\x47\x7b\x9b\x70\xa1\x0f\xa9\xa4\x94\x29\x65\x23\x5b\x9e\xa4\x6e\xa2\xb8\x01\x6a\x02\x5a\x25\xa0\x0f\xd4\x56\x62\x9a\xc0\x23\xf7\x3f\x03\xb5\xfa\x0d\xe6\x59\x8a\xfa\xa5\x47\xeb\x62\x29\x71\x83\x93\x9e\x5f\xab\x8d\xb2\x35\xf4\xe5\xf9\x60\x79\xc7\x1b\xda\x46\xb3\xa9\x4a\x7e\x79\x9f\x97\x73\x8d\xbe\xca\x2d\x34\xd3\x20\xea\x95\xa0\xc2\x68\xfb\xba\x2b\xa6\x0b\xe8\x7b\x51\x7f\x4b\x47\x12\x54\x4a\xa3\xff\x4f\xa7\xc8\x2e\x50\x14\xf5\x73\x3e\x3f\xdc\x0b\x54\x70\x5e\xb9\x3f\xb7\xcc\x70\x35\xc8\x53\xf3\x47\xb1\xa5\xb5\x94\x9f\x5d\xb2\x71\xf5\x02\x0e\xc1\x99\xab\x8d\xc8\xb6\xec\x31\x7a\xe7\x47\x5f\x09\x0e\x23\xed\xb8\xbb\x7e\x66\xe4\x31\xae\x8d\x79\xc0\x40\xbc\x58\xf1\x9d\x61\xde\x99\xe7\x5c\xb6\x70\x1d\xc8\x8e\x77\xf4\xc2\xca\xbc\x4b\x55\x1e\xcd\xfe\x18\x12\x4d\x51\xff\x1a\x2d\xeb\x81\xe0\x13\x0e\xfe\x0b\x7c\x1d\xb5\x51\xf0\xc3\xb6\x2e\x0c\x79\xe5\xee\x73\x95\x4a\x1f\x77\x26\xf7\x31\xff\xdc\x1b\xe5\x8e\xb1\x7a\x66\x1f\x3f\x4b\xc9\x79\x09\x9f\xb4\x93\x1d\x31\x6b\xdb\x15\x91\x31\x30\xa9\xa7\x41\x49\x01\x97\x8f\xe9\x77\x63\xd0\xbe\x8a\xfa\x3b\x19\xbf\x2b\xea\x36\xeb\x52\x8e\x66\xfb\x7f\xac\x84\x5f\x5d\x4b\x69\xf1\x58\x3f\x4c\x13\x59\x35\xcf\x0f\xff\x02\x00\x00\xff\xff\x43\xdf\x18\x28\xeb\x05\x00\x00") func pkgUiTemplatesQuery_menuHtmlBytes() ([]byte, error) { return bindataRead( @@ -260,7 +260,7 @@ func pkgUiTemplatesQuery_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1366, mode: os.FileMode(420), modTime: time.Unix(1575985293, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1515, mode: os.FileMode(420), modTime: time.Unix(1576072716, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -280,7 +280,7 @@ func pkgUiTemplatesRule_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 963, mode: os.FileMode(420), modTime: time.Unix(1575985293, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 963, mode: os.FileMode(420), modTime: time.Unix(1576025253, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -505,7 +505,7 @@ func pkgUiStaticJsBucketJs() (*asset, error) { return a, nil } -var _pkgUiStaticJsGraphJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x7d\xeb\x7a\xdb\x38\xb2\xe0\xef\xf5\x53\x20\x9c\x7c\x11\x15\x4b\x94\x9d\x3e\xdd\x3b\x23\x59\xee\x4d\xc7\xce\xc4\xe7\xe4\x36\xb6\xd3\x97\xe3\xf6\xf1\x07\x91\x90\xc4\x98\x22\x39\x00\x64\x5b\x9d\xe8\xb1\xf6\x05\xf6\xc9\xf6\x43\xe1\x42\x80\x04\x25\xb9\x7b\x66\xbe\xbd\xe4\x87\x1c\x82\x40\xa1\x50\xa8\x2a\x14\x0a\x85\xe2\x1d\xa6\xe8\x23\x2d\x16\x84\xcf\xc9\x92\xa1\xb1\xfd\xf0\xf5\x2b\xfa\xb2\x1e\xed\x89\x2a\x33\x8a\xcb\xf9\x25\x59\x94\x19\xe6\x64\xb4\x07\x65\x67\xef\x3f\x7e\xba\xbc\x39\x39\xfd\xe1\xc3\xa7\xf7\xaf\x4e\x6f\x7e\x7a\x79\x76\x89\xc6\xe8\xdb\x83\x83\x11\x1a\x0c\xd0\x82\x41\xa5\x8b\xd3\x57\x1f\xde\x9f\xa0\x31\x3a\x3c\x38\x38\x18\xed\xed\x55\xe0\xa3\xbf\x0a\x98\x68\x8c\xa6\xcb\x3c\xe6\x69\x91\x87\x24\x23\x0b\x92\xf3\x1e\x2a\x4a\xf1\xcc\x7a\x68\x8e\xf3\x24\x23\xaf\xe6\x38\x9f\x11\xfd\x74\x4e\x16\xc5\x1d\xe9\xa2\x2f\x7b\x08\xf1\x79\xca\x22\x92\xa1\x31\x52\x6d\x47\xba\x10\x10\x7e\x73\xf9\xee\x2d\x1a\xa3\x7c\x99\x65\xe6\x85\x82\x8d\xc6\xba\x17\xf3\xc6\xee\x0c\x8d\x9d\xbe\x6b\x75\x24\x0a\x36\xea\x12\x1d\xe4\xa0\x18\x8a\x16\x5d\xd1\x74\x6d\xda\xd3\x34\xbe\x65\x73\x7c\xaf\xc7\xee\xa0\x96\x60\x8e\xd1\x18\x5d\x5d\x8f\xf6\x74\x51\x9a\xa7\x3c\xc5\x59\xfa\x1b\x09\xbb\xa3\xbd\xb5\x87\x80\x11\x4f\x17\xe4\x35\x8e\x79\x41\xc5\xa0\x04\x1a\xc1\x2a\x18\xa2\xef\x0e\xd0\x73\xf9\xf3\xe2\xdf\xd0\x73\xf4\xcd\x77\xdf\xf6\xc4\xab\xfb\xe6\xab\xff\x0e\x2f\x92\xda\x0b\x28\x9c\x57\x85\xf0\xbc\x80\x67\xf8\x2f\x0b\x86\xe8\xd0\x8f\x11\xe3\xa4\xfc\x11\x67\x4b\x22\x10\xba\x12\x95\x0f\x59\xd0\x43\xc1\xe1\x81\xfc\xb3\x10\xbf\xdf\xc2\xef\xa1\xfc\xf3\xcd\x81\x7c\x9a\x8b\xdf\x17\xf0\xfb\x1d\xfc\x1e\xca\x87\xc3\x04\x5e\x24\x01\x74\x7d\x78\x0f\x4f\xf0\xfb\x6f\xf0\xfb\x67\xf8\x3d\x5c\x41\xf9\x2a\xd8\xbb\xf6\xa1\x95\x2f\x17\xf0\x1f\x81\x95\x8f\x15\xa3\x92\x16\xbc\xe0\xab\x92\x58\x64\x6f\x4e\xb2\xe0\x6a\x46\xb2\x29\x1a\xc3\x14\x89\xd9\x13\x8f\x51\x9a\x38\xd2\x53\xef\x74\x7f\x1f\x66\x75\x30\x40\x17\x84\xa3\x84\x4c\xf1\x32\xe3\x9a\x07\x23\x0d\x44\x3f\x03\x30\x05\x76\x54\x7f\x49\x05\x4b\xde\xa4\x79\xb9\xe4\xba\x96\xef\xd5\xd7\xaf\x40\x51\xd1\x3c\x9d\xa2\xd0\xa9\xc7\xf1\x04\x8d\xc7\x63\xb4\xcc\x13\x32\x4d\x73\x92\x68\x06\x6e\xd6\x42\x87\xc0\xc2\x3e\x28\x0b\xfc\x70\xc3\x8a\x25\x8d\xc9\x0d\x25\xac\xc8\x96\xa2\xb8\x09\xf7\xbf\xd5\xc0\xb6\x34\x43\xc1\x01\x0b\x64\x5f\x92\x50\x27\x14\xdf\x4b\xcd\x83\xe2\x22\xe7\xb4\xc8\x18\xc2\x79\x02\x0f\x38\xcd\x09\x45\x53\x5a\x2c\xd0\x1b\x90\xb9\x09\xa6\x0c\x71\xa5\xa1\xa2\x3d\x35\x51\x95\xb4\xcb\xe1\x75\x4a\xcc\xe7\x1f\x29\x99\xa6\x0f\x9d\x21\xfa\xf8\xf2\xf2\xcd\xcd\xc7\xf3\xd3\xd7\x67\x3f\xf7\xe4\xeb\xc9\x32\xcd\x92\x1f\x09\x65\x69\x91\x77\x86\xe8\x87\x4f\x67\x6f\x4f\x6e\x7e\x3c\x3d\xbf\x38\xfb\xf0\x5e\x0b\xf2\xe7\xbf\x2d\x09\x5d\x45\xe4\x81\x93\x3c\x09\x8d\xae\xb2\x87\xd8\x35\x73\x66\xeb\xa1\xa7\xe1\xbb\x25\xe3\x38\x9e\x93\x88\x92\x3c\x21\x34\x74\xd4\xaa\xd1\x7b\xdd\xaa\x39\xc9\x22\x5c\x96\xa2\x1f\x17\x5a\x57\x33\xd3\x5f\x09\x47\x94\x4c\x09\x25\x79\x4c\x18\xe2\x05\xc2\x59\x86\xf8\x9c\xa0\x34\xe7\x84\x12\xc6\xd3\x7c\xa6\xb5\x23\x43\x69\x0e\xef\x2a\xa2\x4a\x3a\xe2\x3c\x91\xe0\x26\x69\x9e\x20\x72\x47\x72\xae\x54\x19\x05\xde\x34\x4b\xc0\x4f\x54\xa0\x43\x35\xdb\x91\x2c\x9a\xa6\x79\x12\x06\x7f\x82\xb7\x37\xf7\xf2\x75\x80\xf6\x35\xf3\x56\x43\xf9\xbb\xa0\xda\xeb\x82\x2e\xd0\xd8\x81\xa5\x20\xc8\xf7\x37\xd3\x82\x2e\x02\x33\xba\x97\x4b\x5e\xf4\x29\x61\x42\x10\x05\xde\x9c\x3c\x70\x84\x29\xc1\xa8\xc8\x91\xe4\xf2\x82\xa2\x45\xb1\x64\x24\xce\xd2\xf8\x56\xa1\x2a\x5b\x5c\x92\x07\x0e\x75\x9d\x25\x46\x33\x3a\x70\xc7\x74\xca\x08\x87\xd5\x23\x92\xff\x7f\x43\xd2\xd9\x9c\xa3\xbe\x28\x89\xb3\x94\xe4\xaa\x64\x04\x6d\x9e\x8a\xf6\x51\xcc\x58\xd8\x99\x43\x71\xa7\x87\x3a\x78\xc9\x8b\x4e\xbd\x94\x64\x11\x8b\x69\x91\x65\x0a\xe0\xbe\xea\x4b\x2f\x07\x66\x7e\x1f\x4a\xea\xa7\x07\x57\xd8\x5f\xe5\x78\x41\xc6\xa2\xde\x75\x60\xf1\xc5\x43\x49\xa3\x5b\xb2\x2a\x29\x61\x2c\xac\x86\xa7\x47\x17\x17\x39\xe3\x88\x08\x16\x10\x12\xfc\x8d\xc4\x5f\x08\x30\x89\xee\xe7\x69\x3c\x47\xe3\xb1\x7a\xfd\xec\x19\x7a\x42\x22\x36\x4f\xa7\xfc\x3f\xc8\x4a\x03\xa8\x4f\x5a\xc4\x96\x93\x45\xca\xc3\xee\x48\xbd\x26\x51\x49\x81\x51\x4e\xa4\x2a\xd3\x6f\xd6\x8a\x52\xb0\xf8\x45\x45\x1e\x76\x6e\xc9\x6a\x59\xca\xd9\xea\xf4\x50\x42\x26\xc5\x32\x8f\x49\xd8\x58\x3b\x51\x6d\xde\xaa\xf5\x13\xa1\x75\xcf\x67\x6e\x48\x41\x59\x77\x5d\x7a\x46\xc0\x0a\x1e\xa2\xb4\x81\x6f\x02\x80\x45\x5f\x8a\x9c\x6d\x06\x58\xf5\x12\x92\x2c\xcb\x1f\x78\xae\x25\xa1\x22\x94\x62\x67\xa8\x70\x33\xe1\xb9\x3d\x6b\x39\x9e\x64\xe4\x44\xbc\x69\x6b\x07\x64\x92\x73\x0e\x10\xec\x49\x2f\x31\x15\xab\xd2\x39\x61\x65\x91\x33\xb2\xa9\x77\x55\x55\xe8\x57\xa8\x5b\x43\xa4\x06\x69\x07\x64\xea\x00\x6d\xbc\x60\xd1\x39\xb3\x97\xa3\x0d\x80\xac\x15\xca\x86\x21\xf4\xe2\x2d\x49\x36\x8d\x49\x55\xa9\x0d\x45\x95\xee\xd0\xb3\xaa\x69\xf7\x9a\xe6\x8c\x50\xfe\x8e\x70\x9a\xc6\x6d\x10\x18\xc9\x48\xac\x40\xc8\xfa\x37\x0b\x68\xe0\x90\x80\x4c\x29\x61\xf3\x33\x21\x51\x77\x38\xdb\x05\x96\x6a\x62\x43\x59\xe0\x87\x0b\x58\x17\xcf\xcd\xb2\xb8\x91\xac\x36\x38\xef\xa2\x6a\xd1\xd9\x70\xae\x50\x0d\x45\x46\x2e\x61\x6d\xf7\x29\x62\x55\x21\xa8\x2d\x62\xa2\x01\x6a\x69\x22\xb5\xbf\x59\x4f\x02\x4b\x50\x38\x9e\x30\x7f\x2b\x7c\x25\x0c\xde\x3e\x2f\x66\xb3\x8c\x8c\x3b\x1c\x4f\x3a\x36\x31\x44\xc3\x88\xfc\xbd\x61\xb7\x74\xc5\x4f\x18\xb0\x79\x71\x5f\xaf\x5d\xe4\xb2\x3c\x8f\x26\x50\x35\xe8\xa1\xa6\x16\x10\x8a\x9f\x63\x3a\x03\xc5\xff\x34\x24\x91\x7c\x50\x8a\xc6\x63\xff\xc8\xf7\x42\x66\x48\xce\xc3\x6e\x94\xe6\x09\x79\x08\xed\xfa\xb6\x8e\xd0\x2f\x84\xae\x7d\x1a\x06\x7f\x12\x6b\xa1\x82\x80\x39\xa7\x61\x80\x69\x8a\xfb\xda\x9e\x09\xba\xdd\x68\x8e\xd9\xab\x0c\x33\x16\x06\x94\x64\x05\x4e\x82\x6e\x4d\x09\x4b\xd5\x0b\x56\x87\xad\x65\xd7\x66\x99\x3c\x27\x7c\x49\x73\x24\x36\x1d\x0c\x4d\x8b\x78\xc9\xd0\x04\xc7\xb7\xc2\x1a\x80\x05\x26\xcd\x19\x27\x38\x41\xc5\x14\x49\x58\xc2\x28\x88\x7c\x42\x10\x4d\x60\x6a\x6e\xc9\x2a\x29\xee\x73\x61\x4e\x53\x80\xed\xa5\x64\xa5\x30\xa1\x4f\x87\x24\x50\x7c\x87\xb3\xd0\x7d\xea\xaa\x3a\x12\x6a\xcb\x22\xb2\xee\x56\xea\x98\xd2\xa2\x65\x81\x94\xef\x82\x6e\x34\x4f\x13\x45\x75\x68\x72\x8f\x69\x2e\x6c\x1e\x7f\x23\xf5\xb6\xd9\x0c\x2a\xbf\x94\xe6\x42\x3b\x8b\x8b\x85\xa3\x2e\x18\x5a\x3a\x0d\x04\xa7\x89\x55\x7b\xf5\xf2\x21\x65\xad\xb5\x57\x37\xf8\x21\x65\x56\xf5\x8c\xcc\x48\x9e\xb4\xa0\x23\x5f\xda\x7a\xb0\x4c\xf3\x9c\xb4\xd1\x4a\xbd\xb5\xd7\xa2\x3b\x9c\x5d\x70\xcc\x5b\x84\x13\xde\xdf\x30\x51\xc1\x96\x66\x92\x27\x27\x98\x13\x7f\x1b\x4b\xd7\x92\x3c\x69\xea\x78\xd5\x58\xec\x73\x89\xd8\xb5\x96\x69\x7c\x4b\x68\x28\x99\x29\x2b\x62\x9c\x91\x21\xea\x90\xbc\x23\x8d\x71\x61\x0a\x62\x3e\x44\x9d\x5f\x7e\xf9\xe5\x97\xfe\xbb\x77\xfd\x93\x13\xf4\xe6\xcd\x70\xb1\x50\xef\x79\x51\x64\x13\x4c\x3f\x66\x38\x06\xeb\x76\x88\x3a\x93\x82\xf3\x42\xbf\x67\x69\x42\x7e\x58\x5d\xa4\x09\x19\x22\x4e\x97\x44\x95\xce\x8b\xfb\xcb\x22\xc1\xab\x1f\x96\x9c\x17\x79\xfd\xd5\xab\x8c\x60\xda\x2c\x2c\x98\x03\x44\x60\xff\x9f\x45\x2e\xd0\xfd\x74\xf9\x0a\xfa\x5b\x77\xbd\x1b\x2d\x43\x08\x57\x68\x2a\x4a\xe0\xb0\x23\xfe\x7b\x99\x2e\xc8\x47\xa0\x47\xa7\x0b\x04\x6a\x03\xa3\x37\x63\x0e\x1c\xa1\xf8\x92\x52\xd9\x2d\xb6\xac\x76\xd1\x17\x9f\x0e\x51\xd8\xfa\x96\x2e\x6d\xfc\x34\x41\x2c\x4b\x81\xd7\xb9\xac\xae\x81\x18\x25\xc2\x2e\xcc\x42\xdc\xb0\xec\x94\xb4\xdb\xeb\xb5\xd4\x06\xb0\x57\xec\x1c\x76\x6a\x56\xf1\xa2\x10\xf3\xb9\x95\xc9\x64\xb5\x26\x9f\xc9\xf2\x3f\xcc\x66\x43\xc6\xfe\x6f\xe2\x34\x51\x93\x71\xbc\x28\xed\x85\x2e\x91\xc2\x9a\x93\x7b\x74\xd2\x60\x2a\xd3\xe2\xf9\xe1\xc1\xc1\x41\xb7\x62\xcf\x8a\x80\xad\xdc\x29\x7e\x24\x2f\x22\x92\x31\xe2\xf1\x0d\x58\x93\xe3\xf0\xfe\x0e\xc0\xdb\x01\x39\xdc\xaf\x20\xfd\x2e\xe6\xd7\x8e\x1c\xbe\xca\x08\x70\xae\xb4\x3c\x1b\xac\x2b\x2a\xa5\x71\x61\xac\xd2\xca\x4e\x95\xfc\xd8\x89\x66\xd9\xaa\x9c\x8b\x2a\x1d\x6b\xe5\x77\x65\x22\x6c\xac\xe8\x15\x14\x9c\x24\x6a\xf5\x9f\xf0\xbc\x5f\xd2\x74\x81\xe9\x2a\x30\xdb\x2c\x01\xd8\xaa\x63\x3a\xeb\xc7\x73\x12\xdf\xd6\xea\x51\xf0\x3b\x36\xaa\x2e\x73\xa8\x4c\x12\x5d\x5d\xcd\x59\x1b\x4a\x0e\x98\xc7\x61\xd5\xe8\x6a\x33\x66\xce\x20\xd6\xda\xc1\xe2\x4c\x4a\x68\x29\x19\x0b\xc7\xda\x1e\x4f\xd3\xd7\x47\x7b\xb1\xc3\xad\x96\xdc\x7f\xbf\xf8\xf0\xbe\x9a\x8d\xc1\x00\x9d\x4d\x2d\x97\xc8\x3d\x66\x48\xf5\xd2\x83\xe2\x82\xa6\xb3\x34\xc7\x19\x62\x84\xa6\x84\x21\xf0\xd1\xce\x0a\x8e\x16\x4b\x8e\x39\x49\x2a\x38\x21\x13\x9a\x25\xe9\x82\x8b\xea\x9e\xa0\x9c\x90\x44\x58\x60\x94\xc0\x66\x9c\x2e\x63\x8e\x52\x2e\x5d\x56\x0e\x64\x81\x11\xc0\x8d\xec\xf9\x50\xce\x60\x69\xdc\x52\x9c\x33\xa1\xa7\x4e\x84\xd0\xd4\xc6\x62\x6f\xbb\x1b\x1a\xb6\x41\x8b\xef\x51\xe7\xa0\x83\x86\x42\xe9\x6a\x73\xad\x4e\x6d\x03\x48\x2a\x7c\x70\x5f\x86\xf6\x46\x79\x30\x40\xb0\x87\xcd\xd2\x18\x0b\xea\x57\x96\x24\x83\xf2\x53\xd8\xe6\x7a\x57\x02\x77\x2d\xb0\xf6\xc3\xde\xf5\xc0\x92\xd1\x93\x6a\xbf\xed\x01\x5a\x97\x52\xbd\x3b\x6f\x95\x51\x87\x53\x6c\xa4\x6d\x51\x7d\x8c\xfc\x3d\x5a\x06\x1b\x52\xf8\x58\xb9\x7a\x94\x6c\xd5\xa5\x4b\x93\x33\xf4\xb8\x34\xda\x64\xcb\x3b\x65\x5e\x12\xb6\x71\x99\xdd\xab\xcd\x4d\x1f\xa5\x73\x01\x69\xe7\x82\xc5\x50\x1f\x5d\x4f\xc5\x06\xd6\xb2\x19\xab\xe6\xdf\xd8\xc6\x5c\x1f\x7d\x8e\x95\xad\x4b\x41\xd3\x1d\xb3\xcb\x92\xe0\x1f\x90\xcd\x76\xff\x0a\x95\xff\x2f\x50\xe0\x4d\xa2\xda\xcc\xe6\x21\xde\x26\xb6\xf3\x4d\xe8\x16\x82\xb6\x31\xa1\x1f\x2f\xd7\x0b\xd8\xe6\x88\xa9\xfa\xdd\x78\x96\x61\x41\x6a\x38\x50\xfd\xe3\xab\x9c\x32\xd6\xac\x68\x77\x80\xa5\x92\xb5\x27\x66\x73\x2d\x9f\x3f\xa2\xcd\x93\xa0\xc4\x66\x8a\x33\x46\x46\xc6\xba\xb4\xb7\x94\x66\xa7\xdc\x1c\x93\x34\xc9\x27\x60\xdf\x6a\x0f\x59\x7c\x03\x2e\xbe\xeb\xa0\xeb\x99\x51\xed\x71\x88\x29\xc1\x8c\x9c\x2b\x04\xed\x4e\x37\x01\x4f\xc8\x0e\xc0\x13\xe2\x01\xbe\x2b\xea\x24\x4f\x76\x41\xfc\x34\x4f\x1e\x89\xf6\x16\xc0\x1a\x69\x0b\xf0\xae\x28\x4b\x2b\x78\x17\xac\xdf\x41\xcd\x47\x22\xbe\x1d\xbc\xc6\xdd\x05\xef\xf5\x2e\x79\xf6\x96\x35\x97\x91\x74\x69\x8a\x77\x01\x25\xa5\xd8\x70\x05\x3d\xf4\x85\x93\x07\x3e\xf4\xc0\x03\xbd\xde\x43\x8b\x42\xec\xbc\x82\x09\x99\x16\x94\x04\xeb\x86\x1f\x4a\xbb\xa7\xc4\x5a\x43\x09\x3c\xa5\xf9\xac\x92\x79\x79\x66\x24\x14\x9c\x54\xfd\x9e\x6d\xa7\xf6\xd9\x8a\x4a\x6a\xaf\x69\x5a\x6c\xd4\x5c\xb2\xd6\x26\x69\x33\x8e\x59\xa1\x4b\xc5\x66\xe8\x84\xa6\x53\xe5\x0a\x1b\x0c\x90\x75\x16\x0d\x73\x85\xe6\x29\xe3\x05\x5d\xa9\x3d\xe0\x13\xd8\xd1\x5e\xf0\x82\xe2\x19\x89\x66\x84\x9f\x71\xb2\x08\x03\x55\xa9\xf2\x25\x3a\xd5\x58\xbd\x5a\x0f\xac\xd0\x88\x71\x9a\xe6\xb3\x74\xba\x0a\xaf\xae\xbb\xee\x66\xab\x2c\xca\x65\x86\x39\x39\x03\xfa\x0b\x1d\x2b\xe7\x80\x29\xcd\x60\x96\x38\xcb\xd7\x67\xd3\xa1\xa1\x7a\xd6\xfe\xe0\x81\xea\x10\xde\xa5\x47\xdb\x8a\xcc\xdc\xa3\x78\x59\x38\xa1\xc5\x3d\x23\x54\x34\xb6\x77\xbf\x5d\x41\x1f\x51\x18\x76\xd1\x40\x45\xa4\x88\x26\x4f\x23\xfc\x19\x3f\x84\x95\x25\x26\x50\x2a\x92\x21\x0a\xfe\x7a\x7a\x19\xf4\x4c\xf1\x92\x66\xce\xa1\x31\xda\x47\xc1\x00\x97\xe9\xe0\xee\x70\x00\x73\xf3\x3d\xfc\x8e\x39\x74\x61\x35\x14\xc6\xfc\xe5\xaa\x14\x4c\xfa\x99\x15\xb9\xf5\x06\xe8\xb3\x8c\x63\xc2\xd8\xb0\x1a\xa0\xa8\xd4\x83\xd3\xce\x0b\x8e\xf9\x92\xb9\x36\xa9\x24\xb6\xa8\x23\x6c\x7d\xbe\x64\xe8\xc9\x78\x8c\x02\x05\x26\xa8\x57\xae\xa6\x60\x5e\xdc\x9f\x52\x5a\xd0\x30\x80\x3f\x92\x9f\xd2\x7c\x06\x5e\x86\xc8\x35\x2d\xe5\x3f\xc9\xaf\x6e\xf9\xda\x79\x92\x73\x40\xef\x0c\xb5\x01\x2f\xd8\xce\x50\xc2\x96\x19\xbf\x3a\xb8\x1e\x35\x5a\x24\xe9\x54\xcc\xda\x3b\xcc\xe7\x11\x9e\xb0\xd0\x9e\xb0\xbe\x05\x4f\xf2\x96\x3b\x70\x68\x7b\x3c\x46\xdf\x1c\x34\x47\x0a\x41\x33\x62\x9c\x3f\x49\x3f\x6f\xd8\x18\x11\x42\xc1\x51\x92\xde\xa1\x58\xac\x9e\xe3\x5f\x03\x9c\x11\xca\x11\xfc\xf6\x95\x73\xf8\xd7\xe0\xf8\x88\x71\x5a\xe4\xb3\x63\x05\xe6\xc9\xd1\x40\x15\xa0\x13\xc2\x49\xcc\x49\x82\x02\xb4\xef\x01\x2e\x90\x8b\x78\xf1\x3a\x7d\x20\x49\xf8\xa2\xeb\xad\x13\x20\x26\xf6\x84\x09\x03\xba\x43\x13\x79\x7e\x8f\x26\x84\xdf\x13\x92\xa3\x55\xb1\x34\x4c\x0c\xfb\x49\xb1\x61\x94\x54\x89\xec\x08\x2d\x4a\x32\xb1\x29\x2d\x72\x84\xe3\x78\x49\x31\x27\x12\x24\x34\x01\xd8\x20\x3a\x0b\x38\x83\x8e\xf1\x92\x11\xb4\xcc\xc9\x43\x29\x47\x20\xd5\x89\x9c\x25\x16\x1d\x0d\x92\xf4\xee\x38\xa8\xe1\xdb\x6d\x9b\xfb\x75\xc5\xc3\xe0\x88\x1f\xfa\xf6\x65\xf2\x9f\x9f\xf9\x84\xd5\xe2\xe5\x3d\xd9\xc7\xba\x2d\xde\xa9\x52\x10\xad\x2a\x69\xa7\xa0\x9d\x9a\xd0\x7b\x45\x7e\x93\xc0\x67\x78\x42\xb2\xc1\xcd\x8d\x58\x18\x6e\x6e\x06\x77\x10\xf0\x64\x5a\xb6\x49\xfc\xe3\x64\xfd\x11\x72\xbe\x99\xc8\xf8\x0e\xa7\x99\xa0\x10\x92\x67\x97\xec\x89\x2b\xed\x75\x39\x5f\x57\x62\x57\xe2\x19\x79\x55\xe4\xd3\x74\x16\xe1\x2c\xab\x28\x6c\xe4\x1c\x96\x55\x5e\x24\xc5\x10\x25\x85\xf1\x7c\x00\x3e\x55\x83\xef\xd1\x07\x8a\x62\x9c\xa3\x94\xa3\xcf\x4b\xc6\x51\x96\xde\x11\xc1\xb8\x82\xb3\x45\x17\xa6\xbf\x69\x41\x51\x08\x7b\x2d\x88\xd3\x42\x29\x3a\xf2\xe3\x10\x65\x24\x9f\xf1\xf9\x08\xa5\xfb\xfb\x1e\x5a\xd8\x86\xc2\xd5\xc1\xb5\xb1\xd8\x71\x92\x84\x62\x45\xf8\x00\xcf\xa1\x17\xf4\x55\x7a\xdd\xf3\x77\x7a\x95\x5e\x77\xbb\x5e\x3a\x41\xa7\xd3\xe5\x6f\xbf\xad\xce\x41\xa2\x4c\xcc\x91\xfc\x07\xc2\x36\x84\x80\xbf\x9e\x43\x78\x51\xb7\x59\xbe\xc0\xe5\x10\x7d\x59\xb7\x76\x24\xac\x02\xc1\x5f\x78\x4e\xb0\x0c\x0e\xaa\x76\xfa\x1a\xce\x26\xb9\xfc\xfd\xec\xb2\xd6\x3e\xe8\x2d\xd2\xe9\x60\x68\x4b\x24\x20\x0b\xa8\xc8\x28\x15\xb9\x7d\x42\x63\x49\xa2\x37\xd2\x22\x89\x52\x66\xef\xe4\xac\xb9\x30\xb5\x34\x1b\xc4\x45\x1e\x63\xee\x9f\xc8\x2e\x1a\xfa\xe7\xd1\x8d\xa4\xe1\x86\x92\x92\x42\x78\xc9\x8b\x0b\xb0\x44\x87\xd2\x56\x53\x1e\x7a\xc0\x74\xa8\xfe\xca\xb2\x94\x93\x05\x1b\x82\x31\x21\x0b\x16\x98\xc7\x73\x62\xd1\x1d\x85\xa2\x4e\xdd\xe7\x78\x4f\xd0\x1c\xdf\x11\xc5\x00\xc0\xf5\xf1\x92\x52\x92\x73\x49\x87\x1e\x62\xb7\x69\x59\x77\x56\x59\xfc\x25\x09\x01\x2a\x01\x56\x3d\x78\x6c\x4c\x71\xb3\x81\x5d\x7d\xd4\x5e\x79\x81\x4b\xc1\xc1\xeb\x0d\x55\xa8\xe6\x73\x28\x8c\xa6\x69\xc6\x09\x0d\x2b\xe8\x91\xb2\xe0\xc3\x01\x1a\xcc\x7a\x28\x08\xba\x3d\xb5\x40\x4b\xfa\x39\xf2\x51\x52\xa1\x2b\xf5\xba\xeb\x58\x48\x65\xc1\xb8\x78\xa7\xd7\xe0\x6a\x8d\x5a\x77\xb7\xa2\x17\x4d\x0b\x7a\x8a\xe3\x79\x65\x9e\x53\x8f\xb2\xa8\x8d\xfc\x8a\x46\xda\x3d\x7b\x8d\xc6\x88\x8e\x3c\x3d\x1a\x89\x54\x36\xbd\x98\x64\x94\xe6\x5e\x78\x3a\x88\x69\x4f\xb1\x11\xe5\x4d\x06\xb1\x14\x3f\x3c\x46\xa2\x5a\x85\x35\xee\x4d\x6c\xbc\xb5\x82\xf4\x62\x3f\xb9\x8e\x58\x5c\x50\x69\x4a\x79\xde\x63\xf5\xbe\x1a\x96\x1e\x03\xf8\xc8\x0e\xd0\xf7\x08\x47\xf2\xa4\xec\x55\xb1\x28\x31\x25\xe1\x44\x48\x52\x6a\xc6\x6e\xa8\x60\x0d\x9e\xb9\xa1\x09\xc0\xe8\x97\xf3\x94\xc1\x7a\x00\xa1\x89\x73\x88\x65\x44\x78\xca\x85\x59\xc3\x39\x8e\xe7\x60\x01\xcc\x09\x32\x12\x88\xca\x6c\x39\x4b\xf3\x1e\xc2\x0c\xa5\x5c\x42\x29\xf8\x9c\xd0\xfb\x94\x11\x34\xa1\x04\xdf\xb2\x5a\x0b\x4d\x23\x9c\xa5\x7c\x15\xed\xb5\x04\x26\x38\xda\x65\x92\xe6\x89\xfa\xff\xe9\x1d\xc9\x39\xd3\x2a\x74\xbd\x51\xa7\xcd\x08\xff\x60\x22\x4a\xb7\x9b\x18\xb5\x08\xd4\xf5\xc8\x0d\x4b\x05\x97\x92\x8e\x91\x46\x28\xb0\x42\xa3\x14\xff\x07\xe6\xdc\x57\x17\x30\x4e\xca\x7a\x09\xf8\xf6\xf5\xa3\x7d\x5c\x26\x04\xe5\xba\xdd\x93\x20\xeb\x74\x23\xe2\x88\x07\x84\xb7\xf4\x74\xc8\xa8\xbd\xd5\x12\x96\x4e\x15\x6a\x1f\x89\x47\x2b\xd6\x25\x4a\xf3\x97\x94\xe2\x55\x28\xca\x7b\xce\x10\xbb\xc2\x5c\xb7\xac\x75\x88\x43\x54\x50\xc0\x6e\x52\x4b\x39\x3a\x46\x8e\x4d\xaf\x68\x07\x7b\xef\x6b\xab\x67\x68\x63\x3b\xb6\x9d\x80\x98\x6d\x61\xc6\xdb\x3d\x7c\x2e\x1c\x15\xa3\x59\xdb\xdc\x8e\xac\x1a\x32\x4e\xa8\x1e\x3a\x24\x7d\x04\x20\x1e\xe6\xaa\xc1\x36\x8b\x16\x53\x46\x4e\x84\x21\x2f\x51\xad\x74\x96\x60\x8d\x4b\xf2\xc0\x2b\x5e\x83\xa2\xf3\x53\xb5\xbf\x3d\x27\xb3\xd3\x87\x32\x0c\xfe\x2b\xbc\x3a\xe8\xff\xe5\x7a\xbf\x1b\x5e\xad\xee\x93\xf9\x82\x5d\xef\x77\x9f\xca\xc5\x5b\x34\x92\x8b\x93\xe0\x39\x03\x31\x82\xb2\x50\x81\x33\x67\xcb\x4f\x54\xd5\x2e\xfa\xa2\xad\x43\x13\x04\xae\x5e\xe9\x59\x7b\x32\x46\xdf\xd4\xbc\xf0\xdf\x1d\x68\xe7\x81\xe8\x15\xe6\x0b\x8d\x11\x0c\xef\x2c\xe7\x1a\xc0\xd5\xe1\xb5\xc1\x6c\x99\xa7\x62\x29\xd1\x6f\x5e\x5c\x5b\xe4\x93\xed\x9f\x37\xa3\xeb\xad\xbb\x0f\x57\x02\xc0\xf5\x0e\x56\x89\xe5\x1d\xdc\x59\x88\x81\x38\x17\x6a\xd3\x56\xb9\xff\xab\xb9\x92\xab\x73\x15\x08\x69\x05\x3b\xf9\xec\xd9\x0d\x57\x26\x7c\x56\xad\xa0\xb9\x83\xc2\x91\x0f\x85\x0d\x40\xc1\x6a\x75\xcf\x7b\x6b\xb8\x6e\x69\xdc\x38\x35\x6b\x7a\x79\xd0\x06\x27\x73\xb5\x71\xb4\x37\x1a\xeb\x5d\xbc\x40\x8e\x3b\xf7\x5f\x3f\x61\xdb\x67\x0a\xf5\xd1\xa1\x98\xd5\x63\x39\xbb\xfd\x7e\xeb\xac\x1d\xff\xff\x33\x6b\x33\xc2\x4f\x4d\xa8\xd8\xf6\x29\x03\x85\xe3\x04\x98\x7d\xfd\x8a\x9c\x02\x17\x6b\xaa\x03\x1e\x95\xc7\x59\xe9\x1a\xf7\x6c\x79\x7b\x88\xd5\xf6\x4d\x8c\x58\xf0\xe9\xc5\xe3\x06\x63\xc5\xdd\xc8\x33\x1b\xd3\xdc\x0a\x37\x64\x55\xa1\x09\xa5\x51\xe8\x27\x70\xc5\x6e\x0b\x62\xcc\x8b\x13\x80\xda\x78\x4b\x69\x17\xb2\x28\x84\x76\xd4\xa4\xa7\xb9\xe7\x38\xb6\x85\x2c\x39\xb9\x57\x28\xab\xa9\xd3\x04\xb2\x89\xac\xc4\x50\xd5\x85\xfd\xfa\xce\xf2\x8b\x06\xe8\x45\x0f\x75\x94\x7f\xad\xe3\xa5\xb7\x02\x6c\xbd\x73\x59\x7f\x47\x85\xf4\xcf\x1e\x37\x5b\x4e\x38\xc5\x31\xff\x3f\x6a\xf0\x33\xc2\xdf\xe9\xe0\xbc\xc7\x88\xb5\x8a\xe8\x33\x52\xad\x42\xb7\x1e\x2b\xd4\x3b\xc4\x8e\xed\x2e\xd3\x8f\x18\x88\x47\xa4\xdf\x59\x68\x6a\x22\xab\xb2\xdf\x2b\xd0\x4d\x84\xb6\xcb\xf3\xee\xa1\x7a\x3b\x8a\xf3\x23\xa9\xb2\x99\xb3\x35\x91\x1a\x02\x7d\x78\xd0\xc6\xa8\xaa\xc9\x3f\x48\x48\xff\xf9\xa3\x31\x62\xfa\xcf\x1e\x92\x55\x7b\xf7\x4b\xa9\x71\x46\x30\x95\x2e\xbe\xae\x5b\xa8\x0f\x48\xba\xb5\xf5\xb7\x61\x21\x54\x6b\xff\x7a\xaf\x1e\x19\xc0\xe6\xc5\x7d\xe8\x89\x29\x8f\xc8\xa2\xe4\xab\xd0\x8e\xb3\xc4\x94\x6f\x38\x8e\xfb\x47\xd8\x6d\xea\xe6\x5f\xb5\xd1\x33\xdb\x8d\xf6\xdd\xaf\xbe\x19\xa4\x37\xd5\xd7\x41\x57\x8f\xfe\xeb\x57\x79\x3c\xb5\xc0\x0f\x21\xfc\x67\x9a\x15\x05\x75\x2d\xba\x01\x7a\xf1\xed\x41\xb7\x87\x0e\xad\x0d\x56\x63\x5f\xb9\xdb\x96\x53\xb5\xaf\x02\xec\x1b\xb6\x83\x75\x64\x09\x3d\x69\xd6\xae\x6b\x24\x53\xcf\x3e\x95\x85\xd1\xff\x3c\xa7\xce\x99\xac\x2e\x8c\xf0\xa4\xa0\x96\xca\x85\xfd\x18\xcd\x4c\x8c\x94\x3c\x70\xd0\x8f\x25\xa6\x78\x51\x5d\xa9\x0d\x00\x4a\x30\xac\x6f\x90\x4d\x48\xb7\xac\x2f\xc3\xcb\xd0\xb8\x25\xea\x0e\x7d\x8f\x3a\x9c\x2e\x09\x04\xec\x80\xcb\x55\xca\x90\x6a\x5c\xbf\x7e\x66\xc1\xd9\x14\xff\xd3\x84\xb8\xe9\x32\xb4\xf1\x3d\xa8\x4e\x81\x69\xd1\xd8\xcc\x49\xdf\x61\xcf\x91\x5d\x55\x5e\xd4\x50\x15\x47\x2e\x10\x22\x46\x5d\x31\xa6\xf3\xb6\xcd\x47\xe1\xe1\x15\xd9\x6e\x49\x33\xb1\x2b\xd9\x70\x06\x2d\x23\x64\x02\x15\x03\x21\xa7\xce\x56\x18\x9e\xf3\x26\x3b\x6a\x08\xd4\x8e\x26\x66\xb3\xf2\x48\x46\x78\x39\xe1\x64\x6a\x2c\x5c\x8a\xb7\xe4\xcb\x5d\xb1\xfd\xdd\x78\xbe\x92\x11\x53\xdb\x31\xb5\x22\xce\x24\xdb\xca\xff\xd4\xdc\x62\x3f\xcf\x29\x1a\xb7\x9d\x09\xd6\xf4\x87\xbc\xc7\x25\x5f\x06\x5d\xe7\xac\x70\x49\xb3\x6d\x27\x80\xa2\x7c\xa8\x90\xf8\x57\x9f\x0a\x42\x2b\x38\x16\xda\x72\xfa\xd7\xe8\x4a\x1d\x8d\xb3\x16\xf0\x7a\x35\x71\xeb\x7a\xcf\xc9\x1c\x3f\xac\x9c\x4b\xf5\xe4\x1e\x63\x19\xaa\x84\xe6\xa4\xd1\x9d\xdf\x6d\xc7\x5d\x0f\x73\xda\x83\x90\xd2\x3a\xed\x44\x19\x7a\x32\x46\x01\xa8\xbd\x1a\xc5\x40\x09\x53\x6a\x93\x47\xb4\x79\x98\xd3\x48\x2b\x1f\x08\xd8\x7e\xe2\xcb\x9e\xa0\xff\x11\x2a\xb8\xa9\xde\x46\x52\xde\x86\xec\x09\x01\xb6\x1b\xcb\xf9\xbd\x24\x0f\xdc\x69\xb4\xf5\xd4\x97\x3c\x90\x78\x09\x17\xff\xd5\xa9\x63\x80\xf6\x05\xd8\xc6\x21\xbb\x45\xbd\xb8\x58\x94\x19\xe1\x64\x67\x02\x8e\x5b\x08\xd8\xce\x4c\x60\x45\x57\xce\x4d\x6f\x58\x4e\xbf\x32\x15\x46\x4e\x43\x5e\x70\x9c\x89\xe2\x0b\x19\x88\x0f\x39\x3c\x36\xcd\x90\x8c\xa0\xdf\x30\x4d\xad\x8d\xd4\xc9\x91\x10\x5e\x58\x17\x02\x16\xe3\x0c\xd3\x46\x60\x4d\x13\xa5\x43\xcf\xe4\xa6\xd3\x8d\xbd\x00\x86\xf9\x32\xcb\xb6\x43\xdf\x04\x46\xbb\x0d\xbd\x7c\xb2\x76\x5d\x3d\x95\x99\x36\xe7\x8b\x2c\x0c\xde\x16\x58\x86\x8b\x48\x46\x31\x53\xb4\x8f\x82\x05\x43\x47\x13\x8a\x06\xc7\xa8\x5a\x88\x64\x2d\x6b\xb9\xda\x47\x81\xae\x26\xde\x04\x97\x02\x73\x19\x7f\x22\x6f\x4d\xc8\x16\xb5\x01\xd5\x8f\xee\xea\xd1\xa6\x15\xea\x3b\x1c\x36\x1b\x11\xb0\x57\x90\x05\x9b\x6d\x71\x86\x88\x16\x91\xd0\x29\x50\xb7\x56\xae\x8d\xdb\x6d\x61\x6a\xc6\xc4\xde\xdd\x2c\xb7\x3a\xee\x74\xea\xfd\x6a\x02\xec\x30\xe4\x9f\xcc\x5d\xd7\xdd\x07\xad\xb4\xb3\x9c\x7b\x67\xd8\xfa\xcd\x63\x06\xee\xc1\xe0\x11\xdd\xdb\x83\x37\x2f\x76\x1b\xbe\x73\x1b\x71\x87\xee\x6d\xc3\x4f\xb0\x66\xb1\xe4\x67\x27\x5a\xe6\xee\xd3\x3c\x29\xee\xe5\x88\x2e\xe5\xcb\x7a\x4d\xb3\x01\x4a\x6b\x77\xfc\x7d\xdb\x93\xda\x95\xca\x6a\x8f\x02\x1b\x2d\x0d\xc1\x3d\x5a\x31\x37\xd9\x75\x97\x68\xac\xf1\x62\x52\x3d\x0a\xac\xfc\x41\xa6\x1e\xe7\x2d\x94\xd7\xaf\x6c\xee\x41\x26\x0d\x33\x82\xe7\x2a\x3d\xd7\x76\x6a\xcb\x7c\x35\x6f\xf1\x84\x64\x8e\x91\x06\x41\x50\xac\x22\x39\x3c\x5f\x40\x14\x29\x53\xa9\xac\x2c\x87\x3a\xbc\x45\x69\x8e\xec\x66\x92\x28\xf2\x95\x58\x94\x75\x44\x95\xa5\x6e\x6d\xa8\x51\xb9\x64\xf3\xb0\x0a\x14\x40\xfb\x0a\xec\xbe\x15\x21\xa0\x56\x3c\x16\xe3\x92\xbc\xb9\x7c\xf7\x56\xe1\x79\x05\x7f\x4c\x00\xcf\xda\xf5\x30\x65\x7a\x74\x6e\x80\xa0\x2c\xfe\x35\xa8\xba\xd2\x98\x7c\x2e\xd2\x3c\x0c\x8e\x26\xf4\x38\xe8\xca\xee\x21\x82\x6e\x2b\x31\x65\x4c\xcd\x65\x71\xc9\xde\xcb\x13\xd5\x56\x72\x72\x5d\x43\xbd\x89\x34\x71\xc4\xee\xb4\xd3\x81\x5e\xbf\x04\xa3\x4d\xc4\xdf\x4a\xfd\xed\xe4\xf7\xd0\xdf\x90\x7c\xfc\x6b\x60\xe8\xa2\xe9\x2b\xca\x7f\x0d\x4c\xc4\x10\xac\x3e\xe2\x47\x8d\x66\x7f\xec\x23\x63\x4f\xd2\x70\x1d\x58\x8e\x33\xd9\x60\xb7\x53\xd3\x1f\xd5\x19\xa3\xa1\x25\x1c\x1a\x56\xa4\x94\x12\x0b\x55\x5f\x67\x05\xe6\xea\xbd\x16\xca\x94\xbd\xc7\xef\x45\x99\xf1\x7b\x0c\x06\x28\xd8\x3f\xcb\xa7\x41\x0f\x05\x7d\xf5\x17\x9e\xd1\x7d\x9a\x65\x68\x42\x24\xb0\x44\x88\x53\x81\xde\xe3\xf7\x68\xb2\xb2\xe1\x77\x23\x74\x39\x27\x1a\x54\x8c\xf3\x0e\x17\x8d\x20\xb6\x9c\x24\x3d\xc4\x0a\xb8\xf1\x8b\xf8\x9c\x2c\x10\x66\x68\x86\x4b\x86\x42\xb0\x04\x22\xdb\x1f\xaa\x53\xc4\xad\x9d\xf3\xd0\xad\x44\x71\xee\x0c\xd6\xf7\x55\x1b\x9d\x60\x25\xce\x08\x37\xd7\x86\xcf\x55\xc6\xba\xe8\x55\x91\x15\x34\xfa\x28\x5f\x56\x6e\x23\x30\xce\x2d\x83\x49\xf0\xd0\x02\x73\x9a\x3e\x04\xae\x8a\xaa\x8c\x54\x15\x1f\x97\x32\x94\x17\x1c\x15\x53\x24\xeb\x43\x44\xc7\x13\xf4\x31\x23\x98\x11\x95\x9c\x08\xa3\xb8\xa0\x94\xc4\x1c\x12\x59\x10\xc6\xd2\x22\x37\xc1\xa2\x8a\x1a\x92\xcf\xd7\x95\x9b\x16\xeb\xe8\x44\x6a\x42\x5e\x2a\xbd\xc9\x59\x3d\xa4\x61\x64\x9e\x24\x17\x57\x31\x0d\x9c\x29\x59\x05\x33\xd0\x35\xd2\x54\x30\x84\xb6\x0d\x47\xb6\xaa\x62\x56\xe4\x54\xcd\xc4\xd7\x31\x14\x95\x6a\x92\xbe\x21\x47\x25\x54\x1d\x57\xb1\x86\x06\xb0\x79\x57\x29\x31\x43\x0a\xbb\x97\x21\xfc\xf6\x9c\xe6\x43\xf5\xd7\xdd\x8b\x72\x26\x23\x2a\x98\x4b\x29\x4b\x80\xe4\xbf\x5a\x27\xe2\xdf\xc3\x50\x1e\xce\x5f\x1d\x5c\xdb\x11\x5b\xab\xa1\xb5\x36\x82\x64\x4a\x68\x57\x87\xd7\xdd\xca\x2a\xad\xa2\x89\xaa\x4d\x48\x26\xb6\x70\x8a\x03\x23\x78\x0c\x65\x0b\xb9\x99\x07\x72\x80\xd9\xdb\x08\xeb\x62\x96\xe0\xca\x88\x60\x98\x31\x06\x0a\x10\x67\x19\x5a\xa4\x8c\x09\x53\x85\x71\x52\xb2\xa8\x62\x01\x72\x6f\x2c\x6c\xa5\x32\xa5\x18\x14\xd6\x26\xc3\x28\x51\x6e\x2d\xfb\xc6\x47\x34\x42\x1c\x1d\xb9\xe5\x24\x4f\x44\xe9\x7e\xbd\x36\x29\x9d\x40\xc0\x97\x59\x56\xdc\x03\xf4\xa9\x50\x1a\x02\xbd\xb2\x48\x73\x8e\xd2\x5c\x46\x74\xc7\xab\xc8\x3e\xc4\x95\x26\xbf\x89\x96\x11\x38\x3e\x7b\x86\x64\xf1\x55\x59\xb0\xeb\xe8\x01\x1d\x89\x7e\x1b\xdd\x4a\xaf\xa0\x3d\x9d\x66\xe0\x52\xa5\x5b\x40\x2c\xd3\xbc\x2c\x20\x73\xa1\x9a\xa8\xfa\x76\xb5\x06\xe2\xcb\xc3\x10\xf1\x1e\x52\x61\xae\xeb\x6e\x33\x44\x07\x21\x93\xe6\xd2\xb4\xad\x26\xb6\x3a\x2f\xc1\x3b\xda\x7f\x8d\x1c\xa2\x1b\xcf\xa2\x80\x26\x36\x05\xb5\xd7\xcf\x35\xc3\x20\xa3\x0e\x64\xf8\xc4\xf9\x0a\x71\x8a\x63\xc2\x84\x9a\xc2\x39\x22\x0f\xa9\xcc\xa8\x07\x6a\x3c\x72\x53\xb5\x54\x5e\x6f\xab\xbb\x2a\xcf\x4b\x3c\x4f\xb3\x84\x92\x3c\xec\x7a\xc2\x9d\xaa\xba\xb5\x1b\x43\xf0\x02\x32\xc7\x38\x2f\xd6\xf5\x14\x34\x4f\xc3\x8e\x65\xb6\x04\x32\xf7\xcc\xb1\x34\x49\x3a\xcd\x1c\x34\xb5\xea\x2a\xf9\x4c\xb3\x7e\x85\x7e\x23\x0f\xe1\xb6\x4a\xd0\x55\x75\x04\x40\xf2\x44\x1d\x00\xb4\x7a\xb6\x05\xe5\x5f\x15\xf9\x9d\x90\x5d\x5e\xa0\x4f\xef\xcf\x7e\x46\x26\x49\x85\xce\x43\x68\x79\x10\x76\x3f\x19\xfd\xfa\x15\x7d\xf3\x9d\xea\xe1\x70\xae\xd3\x6f\x46\x9e\xd3\x09\x8d\x66\xdf\x74\x64\x86\xb9\x5d\xef\x7c\xc4\x09\xc4\x4f\xab\x64\x01\xf7\x29\x9f\xa3\x34\xbf\x4b\x59\x3a\xc9\x08\x0a\x84\x54\x04\x52\x61\x32\x84\x39\x84\x31\xc6\x10\x99\xbc\xa4\x24\x41\x0f\x7d\x31\x09\x68\x52\x2c\xf3\x04\x03\x00\x92\xb3\x25\x25\x4c\x83\xe7\x73\xcc\x25\xe7\x31\x84\x29\x41\x49\xca\xca\x0c\xaf\x48\x22\x7b\xc2\x68\x9a\x3e\x54\x70\x80\x0a\x4e\xee\xa7\x1c\x97\x25\x04\x5c\x16\xd0\xb5\x89\xf2\x36\xf0\xc5\xc0\x75\x33\xa8\x52\xa5\x25\xa8\xd4\xcf\xd5\x81\xd0\x32\xc7\x15\xd5\xac\x18\x15\x49\xa3\x65\x0e\xe9\x04\x41\x1f\x98\x5a\x0d\xbd\xb0\xae\xc3\x75\xb5\x5b\x1f\x1d\x4a\x6d\xa6\x66\xa4\xd1\x8b\x51\x39\xaa\x82\xb7\x83\xea\x92\xf9\xfb\xe2\x1e\xc5\x94\xc0\x1d\x99\x39\x01\xdb\xc6\x15\xe2\x46\x6e\x5e\xdb\xfa\x91\x59\x10\x24\x06\x2a\x0c\x71\x68\x31\xbf\x59\xff\x64\x06\xc8\x61\x75\x74\x64\x09\x36\xf8\x37\x64\x42\xc8\xb0\xdb\x03\x75\xdc\x53\xdb\xcf\x84\xcf\x37\xb4\xf9\x49\xbc\x07\xe7\xd8\x9f\x0f\x7a\xe8\x85\x69\x27\x77\x65\x84\x0e\x3d\x49\x2f\xbe\x57\x91\xa1\x01\x1a\xa2\x20\x4b\x73\xa2\x3d\xd5\xb0\xfb\x2b\x8b\x0c\x2b\x5f\x8e\x78\x87\xa9\x72\x4f\x6b\x7f\x8d\xe1\x77\x15\xd3\x9e\x8a\x9a\x78\xc9\x8b\xa0\xe7\x10\xf5\x75\x9a\x27\x70\xdb\x88\x11\xc5\x99\x1d\x86\x16\xf8\x61\xb0\x48\xf3\xbd\x96\x74\x1c\x42\xe9\x72\x5a\x99\x16\x83\x01\xfa\x69\x4e\x72\x9d\x77\x43\xd8\x85\x32\x27\x5c\x62\xd6\xe2\x05\x7e\xa8\xd6\xe2\x0d\xb2\xc8\x2b\xef\x92\x93\x1b\x22\x5e\x52\x2a\xcb\xdf\xd9\x90\x64\x7a\x1d\xb5\x82\xf9\x21\x8a\xd2\x8f\x62\x45\xae\xfb\x40\xcd\x8b\x68\x85\x8e\x6b\x1d\x3c\x7b\x86\xec\xd7\x4f\x7c\x0e\xbe\x3a\x4a\x56\x03\x8f\x97\xd6\x2c\xa5\x82\x12\xfb\x63\xb7\xb5\xe2\x76\x7b\xc1\x70\x78\x39\x92\xe4\x5b\xe0\x87\xe7\x87\xd1\xc1\xb7\xed\xd5\xd2\x5c\xd3\xc6\x59\xe9\x61\x06\xe0\xdd\x59\x3e\x4d\xf3\x94\xaf\x46\xb5\x99\xe9\xbb\x2f\x1e\x39\x43\xff\x98\x49\x38\x02\x1c\x77\x21\xbd\x1c\xcb\x46\x82\xfb\xe6\x78\xb1\xe3\xcc\x2e\x76\x9f\xcf\xb5\x95\x1f\x02\xb0\x1a\xc3\x34\xd5\x83\xfe\xfc\x93\x89\xf6\x2b\x7f\x73\xeb\x6c\x8a\xdf\xbe\xae\xe7\xcb\xfb\xd3\x0e\x3c\x3c\x88\x0e\x9f\x87\xe6\x8a\xa6\x28\xec\x0b\x78\xdd\x6a\x53\xb2\xa5\xdb\xad\x10\xd6\xda\xa9\x26\x58\xe9\x41\x99\x26\x4d\xbd\x1b\x81\xf9\x03\x27\x04\x5f\xa4\x96\x19\xfa\x54\xb6\x75\x9b\x7b\xb5\x05\xd6\x2f\x4a\x95\xb7\x02\x93\x7a\xaf\xa0\x29\xc9\xb9\xd1\x94\x64\xaa\xa3\xee\x79\x1a\xdf\xbe\x56\x89\xc3\xe0\x4a\x8b\xcc\x22\xf6\x1f\xef\x7e\xb8\xec\x79\xd6\x08\x40\x47\xad\x11\xf6\x95\x6f\x97\x74\x2a\x29\x74\x35\x8a\x79\x71\x47\xe8\x09\xe1\x38\xcd\xfc\x63\x79\x53\x55\xd8\x6d\x40\x12\x4d\xe7\xf6\x49\x28\x75\x7e\x0f\x3d\xf4\xd0\xca\x55\x9b\x2a\xe4\xa9\x73\xc4\x4a\x9c\x6b\x53\x51\x14\x06\xc7\x1d\xb4\x5f\x1d\xe0\x3c\xa0\xe7\x60\xc0\x75\x23\x5e\x7c\xba\x7c\x25\x1d\x3b\x61\x17\xed\xa3\xce\xd1\x40\xb4\x3d\xee\x8c\x2c\xb0\xec\x1e\xf3\x78\xde\x04\x0c\xe3\xb8\x91\x6f\x03\x99\xac\x63\x1c\x4c\x70\x7c\x3b\xa3\xc2\x24\xea\xab\xdd\x61\x07\x76\x37\xa0\x2e\xa0\x44\x74\x23\x2c\xd7\x66\x47\x71\x91\x73\x15\x23\x21\xbb\xdc\x47\x6a\xb4\x91\xcf\x9f\x06\x86\x99\x74\xaa\x0d\x91\xed\x60\x5c\xa9\x91\xc8\x12\xd3\x85\x15\xde\x05\x15\x26\x14\xc8\xa2\x7b\xb5\x8a\x94\x57\xb8\xf2\xa1\xba\x68\x34\xed\x15\xf0\x46\xe8\xfc\x8f\x9e\x89\x7f\x0b\xef\xbc\xf6\x88\x6c\x66\x0c\x92\x8d\x0c\x61\xf5\x36\x4f\x67\xf3\x4c\x98\x26\x90\x43\xd2\xd3\xe5\x0f\x64\x8e\xef\xd2\x82\x46\x4a\x55\xbf\xd1\x0d\x42\xb4\x13\xeb\x49\xbc\x86\xea\xaf\xdb\x39\x9b\x93\xec\x4e\x1e\x23\xec\xd0\xf3\x25\x58\x07\xbb\x31\x7c\x5b\xaf\x76\x24\x81\xc9\x93\xb2\xd5\x09\xce\xd2\xdf\x7e\xcf\x96\xd3\x55\x53\xf5\x03\x3f\x8f\x26\x30\x9b\x02\x13\x8a\xf0\x7b\x4d\xc4\x0d\x56\x41\xa5\x6e\x76\x08\xe8\xf6\x04\x87\x6c\x09\xd6\xf0\xd3\x44\xec\xad\x15\x16\x2a\x85\x1a\x43\x25\x66\x0c\x6e\x0c\x57\x19\xd6\xa6\x05\x35\xf6\xa0\xdc\xf0\x80\xc3\xd4\x4a\xab\xc6\xf0\x1d\xd9\x53\xbb\x22\x2b\x99\xda\xcb\x7f\x7f\xf9\xb3\xc9\x23\x25\x76\x31\x05\x4d\x08\x95\x79\xd8\xfa\xc6\x27\x8a\x52\x2e\xdd\xb6\x56\x9f\x12\xd8\xbd\xb0\x44\x05\xc4\x25\x23\x54\x6c\xb0\xc4\xfe\x48\x5e\x38\x03\x7c\xec\xc4\xb9\x26\x07\x9b\xf2\x37\x3a\x1b\x45\x7f\xee\x36\x70\xbe\x6e\x75\x47\x78\xbd\xa6\xef\x0b\x40\x13\xdc\x43\x0c\x4d\x85\x46\xac\x79\x42\x9b\x7e\x81\x4b\x3c\x71\x33\x37\xd9\xc9\x82\xac\x13\x22\x93\xe3\x6d\x27\x2e\xa8\x85\xde\xd4\x02\x56\xf1\x4e\x7c\x20\x03\x11\xab\xac\x47\x9b\xb1\xb4\x29\x2d\xfd\xe1\xfa\x80\xe4\x87\x22\x59\x69\x52\x5b\xe0\xdc\x54\xd6\x37\x90\x1c\x00\xf1\x49\x91\xa8\x24\x86\xd0\xce\x89\x52\x64\xf7\x29\x8f\xe7\x61\xed\xfc\x5f\xdd\x53\xc6\x8c\xa0\xe0\x8e\xc4\xbc\xa0\xc1\x70\xcf\x36\x0f\xcd\x25\x58\x25\xd4\x62\x21\x69\x1e\xcb\xbb\x33\xab\xbb\x57\xce\x92\xe0\x88\xd3\xe3\x23\x9e\xa0\xb8\xc8\xc4\x1a\x36\xee\xbc\xe8\x1c\x1f\xa5\xc7\xb9\x9c\xf0\xa3\x41\x7a\x7c\x34\xe0\x89\xf8\xa1\xc7\x41\xed\xe6\xa5\xbd\x95\x76\xf0\x51\x8b\x41\xba\x48\xb9\x54\x9b\x36\xba\xdd\x9a\x57\xd3\xba\x9a\xe4\x89\x29\x70\x2f\xd8\xc3\x64\x2a\x03\x57\xa7\xf8\x48\xaf\xed\x65\xd7\x9c\x5a\xf9\x5c\xdb\xc6\xb3\x3d\xda\x44\x8b\xe3\xda\xf9\x9d\x04\xa9\x4e\xd9\x04\x2d\x54\x15\xe5\xb9\xbe\x3a\xbc\xae\x5e\xd9\x64\x92\x84\x81\x5b\xa3\x23\x33\x91\xea\x78\xc2\x3b\x91\xff\x8f\x4e\xd8\xdd\xef\x9f\xb0\xbb\xfa\x84\x99\x1b\x7d\x97\xe4\x41\xe0\x1d\x98\xc3\x0f\x83\xde\x67\x89\xde\x67\x74\x84\xee\xf4\xd9\x82\xc6\xed\xb3\x9b\xad\xa1\x82\xb4\x3f\x36\x95\xaf\x3e\x5f\xab\x29\x45\xff\x43\x4c\xb3\x5d\x7e\x20\xa7\x7a\x42\x07\xc7\x81\xeb\x60\xfe\x83\xbc\x64\x61\xb2\x33\x2b\xa9\xd3\x1f\xc9\x4a\xfe\xde\x65\x15\xa7\x27\x7b\x26\xda\x38\xb7\xde\x11\xd8\xd4\x9b\x3b\x82\x2a\x4e\x47\xd6\xa8\xdd\x3e\xbb\x5b\x3a\x55\x0e\xd2\xa1\x77\x25\xfa\x94\xb3\x65\x59\x16\x94\x93\x44\x5d\xcd\x84\x93\xbb\x06\x90\xf5\xef\xb6\xb3\xfc\x9f\xd7\xf2\x25\x71\xa9\x7f\x17\xc7\x71\x92\x5b\x9d\x9f\xfb\x8b\x77\xc6\xa9\xda\xdf\xd9\x78\xad\x2a\xc4\xf0\x84\xdd\xac\xec\x94\x46\x2b\xb3\xce\xcb\x57\xc7\x63\x74\x48\x5e\xfc\x5b\xed\xb6\x4b\xb8\x42\x03\x59\x1e\xf1\xc2\xda\x38\x05\xbf\x04\x23\x37\xbb\xb2\x0d\xe5\xb0\x05\xca\x61\x1d\xca\x7f\x6e\x80\x72\xf8\x67\x3f\x94\xc3\x3f\xd7\xa1\x9c\x6e\x82\xf2\x6d\x0b\x94\x6f\xeb\x50\x3e\x6e\x82\xf2\xa2\x05\xca\x8b\x3a\x94\xcb\x0d\x50\xfe\xe2\x07\xf2\x97\x3a\x8c\xbf\x6e\x80\xf1\x9d\x1f\xc6\x77\x75\x18\xef\x36\xc0\xa8\xdf\x86\x56\x30\xbe\xa9\xc3\xb8\x6d\x87\x51\x83\xb0\xf2\xd5\x73\xd6\xa8\x4d\x15\x8f\x04\x52\xfd\x36\xde\xeb\x37\x99\x6f\xe5\x47\x4c\xc1\x69\xe1\xbe\x7e\x93\xfd\x7e\xdb\x04\xa7\x8d\xff\xfa\x4d\x06\xc4\x1b\xe1\xb4\x70\x60\xbf\xc9\x82\xd3\x8d\x70\x5a\x78\xb0\xdf\x64\xc2\x72\x13\x9c\xbf\x54\xcb\x5b\x0d\x50\x83\x11\xf3\x4d\x70\x5a\x38\xb1\xdf\x60\xc5\xff\xf5\x3f\xdb\xc0\x1c\x92\x7e\x0b\x2f\xf6\x1b\xcc\xb8\x68\xc7\xc5\xc7\x63\x5b\x32\x65\x58\x76\x8c\x93\xc7\x40\x5a\x33\x9b\xc2\x4f\xde\xbd\xfc\xf9\xe6\xe2\xf4\xfc\xec\xf4\xe2\xe6\xfd\xa7\x77\xea\xcb\x91\xd5\xad\x17\xc2\x18\x86\x8b\xe0\x41\x7b\x8e\xb6\xd7\x84\xc7\x73\x2b\x45\x9b\x6b\xda\xed\x43\x02\x36\xe9\xf3\x9a\x2c\xb9\x3e\x45\x4b\xf3\x19\x2a\xf2\x6c\x85\xa6\x29\x65\xdc\xb4\xad\xa1\xb3\x8f\x82\x28\x30\xd1\x84\x2e\xe0\xe3\x5a\xe5\xc6\x4e\x4e\x47\xc9\xab\x41\xb8\x61\x2c\x0a\x16\x2b\xb3\x34\x26\xe1\x41\xaf\x0e\xac\x16\x03\x24\xab\x43\xc2\x12\x99\x46\xa9\x4a\x73\xa4\xae\xe6\x54\xc9\x8e\x86\xe8\x0a\xbc\x41\x72\x27\xad\x9f\xea\x89\x94\x86\x32\x07\x24\x9c\xa7\x86\xde\xac\x92\x1d\x95\x2e\xb2\x23\x4f\x54\x05\x98\xda\xb2\xf9\x11\x37\x2e\xe9\x7b\x6a\xd4\xb2\x44\xf9\x57\x7a\x40\xb6\xca\xb6\x02\x5e\x56\xc0\xed\xd3\xf9\xdb\xea\x8c\xdd\xae\xe5\xb5\xdd\x9d\x0a\xf2\xc8\x70\x5d\x05\x73\x3a\x6f\xf5\xb9\x03\x74\x85\x93\x44\xba\x91\x90\xf9\x4e\xd8\xd3\x30\xf8\x13\x4e\x92\x1b\xf5\x5d\x17\x95\xf6\xd3\xa9\x2d\xbf\x9f\x23\x8a\x7a\xe8\xcb\xba\xdb\xb4\x2b\x6a\xc3\xd7\x03\x6a\x92\x40\x0c\x4e\x85\x7f\x8a\xb9\x80\x3c\xeb\x8c\x60\x2a\x3f\xdd\x16\x04\x35\x99\xd4\x41\x50\x8a\x78\x30\xb7\x1f\xf5\x8d\x17\x3f\x9c\x88\x2d\x27\xd2\x66\x0c\x0f\xbb\xc0\x77\x3c\xec\x3c\xeb\x98\xab\x6c\x15\x8c\x37\x24\x2b\x8d\x5b\xb0\x3e\x98\xbf\xd5\xaa\x85\x76\x28\x47\x1d\x86\x1c\x70\xd5\x84\x85\x16\xa6\x5b\xa9\xa5\xa9\x6c\x53\x4b\x7f\x33\xd1\xe5\x9b\x26\xae\xd2\xc5\x01\x24\x7b\x6a\xbe\x57\x68\x7d\xb1\x4a\x39\xfc\xd5\xd7\x1c\xa5\x99\x2d\x66\x56\x3a\x48\x3e\x9d\xbf\xad\xa6\xb6\x6b\xbd\x96\xe6\x66\x6d\xee\xbb\x7b\x90\x38\x71\xcf\x49\x9e\x27\x85\x4f\xb2\xa0\x8e\x9f\x00\xa6\x92\xb3\xd3\x55\xee\xc5\x66\x84\xaf\x8e\x0a\x31\xce\xc7\x2a\x5f\xb5\x20\xd7\x60\x80\xde\x7f\xb8\x3c\x1d\xd6\x72\x2d\x4d\x08\xba\x25\x25\x87\x8c\x54\xab\x3c\x96\x11\x02\x83\x25\x4f\x33\xa1\x26\xf5\xdf\xb8\xc8\xef\xa2\x59\x31\x04\xb8\x6f\xd3\xfc\xf6\x75\x41\x4f\x4d\xa4\xdd\x86\xa9\x30\x64\xf1\x0b\x2f\xcc\xaa\x5c\x5d\x20\xea\xb2\x4e\x05\x27\xd0\x6c\x26\xe5\x0c\x12\x0f\xd9\xc1\x79\x35\x0d\x20\xe9\x50\xa5\x60\xd2\x11\x32\x7f\x98\x57\x2d\x10\x1f\x26\x9f\x49\xcc\x75\x2e\x3a\x9b\x71\x67\x24\x27\x14\x73\xc9\xbb\xb2\x9a\xa3\x7c\x34\xfe\x8e\x4e\x7f\x2a\x23\xb0\x42\x0b\xb6\x0e\xc2\x96\xdf\x47\x94\xb1\xaf\xcf\xd4\x27\x96\x94\x7a\x05\x1e\xb9\xe0\x98\x93\xf0\xcb\xba\x87\x82\xa0\x87\x64\x3c\xcf\xf7\x62\x4f\x67\x91\x76\xab\xc0\x58\xdc\x69\xcf\x93\x64\x3f\xa0\x74\x63\x6a\xbc\xd3\xa5\x52\xbb\x55\x00\xba\xe8\x8b\x1a\xe2\x0c\xfc\xd9\x50\xcf\x73\x4f\xc3\x4b\xf5\x1d\x16\x88\x7a\x93\xba\xca\xfc\x9b\xa3\xdf\x0c\x34\x5b\x99\x18\x5e\x04\x0f\x30\x49\xdc\x26\xf2\xd0\x0e\x86\x75\x96\xdf\xe1\x2c\x4d\x3c\xfa\x48\xa6\x60\xb3\xf5\x99\x6c\x26\xcc\x0b\x35\xed\xaf\x69\xb1\xf8\x20\x3b\x50\x00\x9a\xdd\xf5\xd0\xc1\x8e\x94\x89\xaa\xde\xe5\xe9\x22\x1a\xa3\xc1\x7f\xcd\x7e\x4d\xf6\x7f\x8d\xa2\xfd\x71\xb4\xff\x74\xf0\x38\x62\x79\x46\x68\xd3\x0b\xb8\xf3\x72\x59\x66\xfa\x38\x5e\x0d\xd3\x2a\x6f\xcc\x7d\xf5\xae\xb6\x04\x3d\x7a\x70\x11\x27\x8c\xdb\xf0\x46\xfe\xcb\x3e\x5b\x07\xb9\x69\x3e\x5a\xd8\xa3\x27\x59\xf6\xac\xd2\x39\x62\xc1\xb5\x2a\x54\xc6\x44\x63\x8f\x55\x5b\x6b\x4b\xf8\x9c\xf0\x87\xa9\xd0\xbf\x00\xcf\x49\xb1\x08\xd0\xe4\x17\x87\x43\xab\x4b\xbd\xc8\xe6\xcb\xc5\x84\xd0\x0f\x53\xd9\xe9\xeb\x82\x0a\x28\x5a\x60\x6d\x74\x76\x9e\x86\xea\x85\x0c\x4e\x65\x3f\xa5\x7c\x1e\x36\x90\x54\xc4\x36\xf7\xc6\x14\x05\x36\xe1\xb3\x9d\x12\xdb\x06\x51\x19\xb7\xed\xfd\x74\xad\x4b\xe3\x35\x50\xcd\x42\x77\x21\xd9\x89\x26\xc6\xe8\x69\x90\x44\xd1\xc2\xb0\x61\x23\x83\x5f\x65\x83\x5a\xd2\xfd\x61\xfa\x21\x57\xeb\x72\x13\x3f\x33\xcf\x12\xc8\xcb\x38\x5e\x2e\x96\x19\xe6\x70\x59\x6c\x07\x65\xd2\xc2\xb1\x68\x5f\xa5\x1c\x68\x80\x35\xb1\x77\xd5\x97\xa8\xeb\x69\xe8\xac\xda\x8f\x16\xb5\xf6\xc1\x6f\x57\xc3\x4e\x22\x44\xe4\x32\x77\x23\x4c\xc8\x9e\xc4\xaa\xf5\x7b\xbc\x20\x2f\xf3\x44\xdf\xf5\xe0\x72\x46\xa5\xe5\x3a\xee\x58\x8b\x79\x55\xdd\x7c\xe8\xdf\x6e\x0b\x39\xc9\x6b\x95\x35\xd0\x84\xc4\x45\x42\x3e\x9d\x9f\xbd\x2a\x16\x65\x91\x93\x5c\xd3\xd2\x01\x70\x78\x5d\x65\x37\xfd\x75\x1f\xd2\x9b\xa2\xa0\xab\xd3\x95\x0b\x49\xb2\x51\x18\xa3\x80\xe3\x89\x75\xa5\xc6\xed\xd2\x24\xaa\xb0\x8a\xe5\xd7\x0a\x38\x9e\xa0\x94\x41\xcc\xde\x8c\x50\xe5\x87\xb6\x2d\xd5\xab\xaa\x9b\x6b\x33\xd4\x1f\x75\x7e\xc4\xb5\x67\xfa\x9b\x59\x08\xb7\x4d\x7a\x5d\x8f\xd9\x53\x6d\x19\x6d\xaa\x97\x60\x26\xac\x94\x54\xb1\x29\xec\x96\x1f\xd9\x9f\xc7\xd4\x6a\x58\x2f\x35\xab\xcb\x70\x59\xa9\x31\xf4\x6b\xe0\xd4\x51\xbe\xae\xc9\x27\xd9\x52\x3e\x46\xb7\x64\xc5\x9c\x9e\xba\x4d\x26\xbd\xad\x3e\x97\x6d\x41\xba\x52\x28\xec\xa3\x5b\xb2\xba\xd6\x76\xab\x82\x72\x25\xca\x1a\x01\xef\x56\x6b\x63\xd3\x5f\xce\x09\x23\x88\xdf\x17\x2a\x0b\x01\x43\x61\xca\x4e\x48\x49\x49\x8c\x39\x91\xfb\x20\x61\x7e\xe3\x3c\x41\x94\x24\x29\x25\x31\xbf\x2c\xde\xa5\x33\x41\xb9\xe4\xd3\xf9\xdb\xae\x80\x82\x29\x41\x38\x49\x48\xa2\x5c\x1b\x05\x85\x2f\xc3\xde\x63\x9a\xc0\x8d\x6f\xcc\xd3\x49\x9a\xa5\x7c\x25\xb6\x0c\x45\xa6\x73\xc2\x4b\x6f\x77\xb4\x67\x32\xd4\xfa\xba\xde\xb0\x51\x9d\x63\x36\xdf\xb0\x80\x56\x9f\xbf\xd0\x3a\x56\x0a\x5d\xf2\x9a\xe2\x99\xca\x6a\xe2\x11\x43\x5f\x2f\xf2\x34\x97\xae\x8c\x64\x59\x3e\x8c\x1a\x50\xa5\xfa\xc3\xc3\xae\x94\xad\x84\x16\x25\x1c\xec\x0b\x38\xe8\x4f\xe0\xfc\x8a\x21\x4c\x28\x24\x0d\x17\x9e\x85\x72\x65\x0c\x52\x21\x65\xeb\xbd\x8a\x4c\xde\x89\xb0\xb6\x45\x7f\x6c\x98\x9e\x3d\xd1\x1f\x19\xad\x5f\x02\xea\x4e\x11\x67\x81\x2d\x5c\xa9\xab\xd4\xb3\x11\x3b\x8f\xf4\x8b\x3a\xb6\x54\x15\xbb\x08\xd4\x66\x91\x2a\x6a\xd2\xa4\xe5\x69\xed\x8a\x36\x64\xf4\xf0\xef\xc0\x6a\x44\xf6\x64\x29\xa9\xed\xb8\x60\xa2\x07\xcf\x9f\xef\xa1\xe7\x48\x26\x61\x52\x59\x2b\xd0\x5c\x6e\x6a\xf4\x28\x98\xa8\xf1\xfc\xf9\x40\xb9\xe5\xec\x74\x17\xca\x31\x67\x52\x9b\x7b\x72\xb3\x2b\xd6\xda\xea\x89\x93\xdf\x61\xeb\x03\xf4\x7e\xe5\x96\xdb\xd3\x59\xa6\x1b\x09\x8e\x87\xde\xed\x9f\xce\xc1\x2e\xbf\x50\x7d\x9a\xc9\x9b\x21\x49\x7a\x17\xd5\x20\x8f\xac\xca\xea\x23\x68\x4f\x43\xdd\xaa\x5b\xdd\x6d\xe9\xa4\xba\xae\x7e\x19\x15\xd3\x69\xd8\x01\x5f\x59\xc7\x5e\x1f\xdb\xb2\xbd\x5b\x87\xee\x42\x85\xcb\x50\xf0\x0f\x79\xd5\xd9\xc8\x5e\x06\xab\x5e\x72\xdd\x49\xcf\x97\xf1\x1e\xee\x78\xc6\x52\x9a\x64\x94\x49\xc7\xf3\xd1\xb3\x8e\x93\x17\xd3\xc1\xb0\x0d\x0f\xc7\x65\xde\xda\x03\xc0\xdf\x0e\x7d\x3a\xf5\x80\xb7\xf9\x1b\xe6\xd6\x49\x9a\xb2\x79\x5e\xb5\x18\x48\xdf\x89\x9b\xbd\xd8\x4c\x27\xbb\x48\x17\xa5\xfe\x78\x86\xbb\xff\xb7\xbe\xb6\x00\x32\xfd\x61\x1a\xaa\x3c\xf3\x62\xd7\xdf\x3f\xac\xc2\x97\x5d\x28\x75\xfd\xe9\xcc\x59\x46\xb8\xba\x3b\xfb\x37\x57\x38\x1e\xe1\x7f\xb6\x07\x30\xc7\x4c\xd4\x54\xe0\xb4\x3d\xe5\x00\xdf\x86\xbd\x0b\xa2\xc2\xde\x03\x49\xed\x61\xb6\xf6\xa1\xb3\x82\x99\x61\x7b\x5a\x80\x33\x50\x56\x1f\xd5\x67\xed\x55\xb1\x84\x15\xc2\xd3\xcc\xce\xf5\xe1\x25\xa4\x0f\x6d\xc0\xda\x82\xdc\x47\xdf\x1e\xf4\xac\xae\xb4\x6c\x7a\x3f\x14\x65\x28\xdf\xf8\x50\x54\xb3\x2b\x3d\x39\x1b\xbe\xd9\xe0\x45\x71\xf4\x58\xd5\x20\x7d\x90\xe6\x1b\x13\x12\xf6\x05\x71\x3e\x05\xf1\xb8\xaf\x45\xd8\x81\xb8\x20\x6a\x5a\xea\x2d\x29\x33\x02\xaa\x90\xf9\xe7\xa1\xa2\x9c\xd5\x2d\x5a\xd6\x09\x99\xf3\x2b\xb3\xea\x2b\x81\x4d\x55\x54\x53\xd2\x55\xcd\x94\xf5\x0d\x80\xd1\x06\x8e\xf0\xae\x40\x3d\x79\xb3\x67\x54\x27\xe0\x74\xfa\x8f\xa0\xa0\x45\x9e\xdf\x4b\x1d\x35\x7a\x2f\x65\x2c\xca\xd5\xa8\xe3\x00\xfb\x83\x04\x02\x4b\xd2\xa2\x50\xdb\xc0\x2d\x82\x2d\x74\x59\x95\xa0\xae\xe1\xa1\xad\xdb\x52\xce\x62\x60\x9d\x95\xc9\x45\x40\xe6\x0b\x35\x9f\x50\x70\x57\xa6\x4d\xf5\x22\xf3\xb5\x16\x83\x94\x77\xa1\xd2\xdb\x98\x13\x32\x25\x54\x6c\x60\xef\xb4\xa5\x55\x4c\xd1\x34\x87\x9d\xc8\x3d\x4e\xf9\x47\x42\xd3\x22\x11\xe8\xc9\x85\x82\x54\x9f\x73\x10\x1b\xdf\x18\x67\x19\x11\x1b\x9c\x92\x08\x6b\x3a\x5b\x55\xb6\x76\x42\x26\xc5\x32\x8f\x49\x38\xcd\x7b\x16\x28\x2b\x79\x86\x4c\x99\x62\xed\xb3\x1a\x76\x81\x93\xf0\x45\xd5\xb7\xbc\x08\x98\x42\x22\x0d\x4c\x67\x4b\x61\x52\x5b\x1f\x98\x63\x71\x51\x12\xe7\xb3\x73\x15\xd6\xf9\x5d\x71\x4b\x5e\x7b\xbe\x79\x3c\xcd\x23\x5c\x96\xd9\x2a\x84\xd6\x3d\x00\xef\x04\x97\x29\x0c\x20\x10\xcf\x64\x7c\xd1\xf0\x9c\x41\x8e\x64\x76\x45\x7b\xef\x61\xa7\x14\x90\xa7\x03\x86\x12\x24\xe7\x29\x5f\xbd\x93\x5f\x74\x81\x9e\x82\x67\xc1\x10\x05\xcf\xf0\xa2\x1c\xe9\x0f\x47\x1c\x41\x49\xc6\x4d\xc1\x31\x14\xcc\x4c\x41\x27\xe8\x0c\x51\xe7\xd9\xdf\x97\x05\x1f\xa9\x0f\xdf\x07\x9d\x40\x14\xfd\xe9\x9b\xbf\x98\x92\x81\x2c\x79\x78\xf1\x7a\xd4\x31\x29\x20\x15\x01\x54\xb8\x81\x42\xaf\x72\xa0\x5c\x3d\x3b\x3a\x0e\x3a\xbf\x0e\xae\x07\xb3\x9e\xf5\x55\x14\x56\x33\x8a\xcd\x30\xae\x98\xd9\x43\xdb\x14\x48\xf3\x94\x2b\xaa\xcb\x44\x7e\x17\x84\x2f\x4b\x15\x42\x1f\xe3\x78\x4e\xd4\xd7\x7c\x2a\xcf\x88\x93\xf0\xcf\xfb\xb5\x2f\xc6\x31\x4f\xe3\xc1\x67\x26\x37\x04\x37\x9c\x2c\xca\x0c\x73\x1d\x0b\x3d\xc1\xf4\xfb\xbb\xb1\xd8\x25\xfc\xf0\xe9\xec\xed\xc9\xcd\x8f\xa7\xe7\x17\x67\x1f\xde\xab\xdb\x94\x8d\xb4\x7e\x42\x9e\x04\x86\x7b\x96\xa4\x5d\x2a\x88\x2a\x44\x53\x8b\xd3\xbb\x25\xe3\x02\x67\xbd\xbd\x13\x2d\x47\xb6\x35\xeb\xdd\x92\xbb\xf9\x1a\xbc\xbb\xd1\x9a\xdd\x6a\x87\x62\x52\xa4\x0e\xf0\x3d\x87\x67\xa1\x95\x15\x00\x3c\x38\x92\xdc\x35\xe9\xaf\xa6\xe5\x69\x28\x2a\x74\x47\x7b\xff\x3b\x00\x00\xff\xff\xd3\x86\x2c\x7a\x63\x94\x00\x00") +var _pkgUiStaticJsGraphJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe4\x7d\xeb\x7a\xdb\x38\xb2\xe0\xef\xf5\x53\x20\x9c\x7c\x11\x15\xcb\x94\x9d\x3e\xdd\x3b\x23\x59\xee\x4d\x27\xce\xc4\xe7\xe4\x76\x62\xa7\x2f\xc7\xed\xe3\x0f\x22\x21\x89\x31\x45\xf2\x00\x90\x6d\x4d\xa2\xc7\xda\x17\xd8\x27\xdb\x0f\x85\x3b\x09\x5d\x9c\x9e\x99\x6f\x2f\xf9\x21\x87\x20\x50\x28\x14\xaa\x0a\x85\x42\xa1\x78\x8b\x29\xfa\x40\xab\x39\xe1\x33\xb2\x60\x68\xe4\x3e\x7c\xfd\x8a\xbe\xac\x86\x7b\xa2\xca\x94\xe2\x7a\x76\x41\xe6\x75\x81\x39\x19\xee\x41\xd9\xd9\xbb\x0f\x9f\x2e\xae\x5f\x9e\xfe\xf4\xfe\xd3\xbb\x17\xa7\xd7\xbf\x3c\x3f\xbb\x40\x23\xf4\xfd\xe1\xe1\x10\xf5\xfb\x68\xce\xa0\xd2\xf9\xe9\x8b\xf7\xef\x5e\xa2\x11\x3a\x3a\x3c\x3c\x1c\xee\xed\x59\xf0\xc9\x5f\x05\x4c\x34\x42\x93\x45\x99\xf2\xbc\x2a\x63\x52\x90\x39\x29\x79\x0f\x55\xb5\x78\x66\x3d\x34\xc3\x65\x56\x90\x17\x33\x5c\x4e\x89\x7e\xfa\x48\xe6\xd5\x2d\xe9\xa2\x2f\x7b\x08\xf1\x59\xce\x12\x52\xa0\x11\x52\x6d\x87\xba\x10\x10\x7e\x7d\xf1\xf6\x0d\x1a\xa1\x72\x51\x14\xe6\x85\x82\x8d\x46\xba\x17\xf3\xc6\xed\x0c\x8d\xbc\xbe\x1b\x75\x24\x0a\x2e\xea\x12\x1d\xe4\xa1\x18\x8b\x16\x5d\xd1\x74\x65\xda\xd3\x3c\xbd\x61\x33\x7c\xa7\xc7\xee\xa1\x96\x61\x8e\xd1\x08\x5d\x5e\x0d\xf7\x74\x51\x5e\xe6\x3c\xc7\x45\xfe\x37\x12\x77\x87\x7b\xab\x00\x01\x13\x9e\xcf\xc9\x2b\x9c\xf2\x8a\x8a\x41\x09\x34\xa2\x65\x34\x40\x3f\x1c\xa2\xa7\xf2\xe7\xd9\xbf\xa0\xa7\xe8\xbb\x1f\xbe\xef\x89\x57\x77\xed\x57\xff\x1d\x5e\x64\x8d\x17\x50\x38\xb3\x85\xf0\x3c\x87\x67\xf8\x2f\x8b\x06\xe8\x28\x8c\x11\xe3\xa4\xfe\x19\x17\x0b\x22\x10\xba\x14\x95\x8f\x58\xd4\x43\xd1\xd1\xa1\xfc\x33\x17\xbf\xdf\xc3\xef\x91\xfc\xf3\xdd\xa1\x7c\x9a\x89\xdf\x67\xf0\xfb\x03\xfc\x1e\xc9\x87\xa3\x0c\x5e\x64\x11\x74\x7d\x74\x07\x4f\xf0\xfb\x2f\xf0\xfb\x67\xf8\x3d\x5a\x42\xf9\x32\xda\xbb\x0a\xa1\x55\x2e\xe6\xf0\x1f\x81\x55\x88\x15\x93\x9a\x56\xbc\xe2\xcb\x9a\x38\x64\x6f\x4f\xb2\xe0\x6a\x46\x8a\x09\x1a\xc1\x14\x89\xd9\x13\x8f\x49\x9e\x79\xd2\xd3\xec\x74\x7f\x1f\x66\xb5\xdf\x47\xe7\x84\xa3\x8c\x4c\xf0\xa2\xe0\x9a\x07\x13\x0d\x44\x3f\x03\x30\x05\x76\xd8\x7c\x49\x05\x4b\x5e\xe7\x65\xbd\xe0\xba\x56\xe8\xd5\xd7\xaf\x40\x51\xd1\x3c\x9f\xa0\xd8\xab\xc7\xf1\x18\x8d\x46\x23\xb4\x28\x33\x32\xc9\x4b\x92\x69\x06\x6e\xd7\x42\x47\xc0\xc2\x21\x28\x73\x7c\x7f\xcd\xaa\x05\x4d\xc9\x35\x25\xac\x2a\x16\xa2\xb8\x0d\xf7\xbf\x35\xc0\xae\x69\x86\xa2\x43\x16\xc9\xbe\x24\xa1\x5e\x52\x7c\x27\x35\x0f\x4a\xab\x92\xd3\xaa\x60\x08\x97\x19\x3c\xe0\xbc\x24\x14\x4d\x68\x35\x47\xaf\x41\xe6\xc6\x98\x32\xc4\x95\x86\x4a\xf6\xd4\x44\x59\x69\x97\xc3\xeb\xd4\x98\xcf\x3e\x50\x32\xc9\xef\x3b\x03\xf4\xe1\xf9\xc5\xeb\xeb\x0f\x1f\x4f\x5f\x9d\xfd\xda\x93\xaf\xc7\x8b\xbc\xc8\x7e\x26\x94\xe5\x55\xd9\x19\xa0\x9f\x3e\x9d\xbd\x79\x79\xfd\xf3\xe9\xc7\xf3\xb3\xf7\xef\xb4\x20\x7f\xfe\xf7\x05\xa1\xcb\x84\xdc\x73\x52\x66\xb1\xd1\x55\xee\x10\xbb\x66\xce\x5c\x3d\xf4\x38\x7e\xbb\x60\x1c\xa7\x33\x92\x50\x52\x66\x84\xc6\x9e\x5a\x35\x7a\xaf\x6b\x9b\x93\x22\xc1\x75\x2d\xfa\xf1\xa1\x75\x35\x33\xfd\x95\x70\x44\xc9\x84\x50\x52\xa6\x84\x21\x5e\x21\x5c\x14\x88\xcf\x08\xca\x4b\x4e\x28\x61\x3c\x2f\xa7\x5a\x3b\x32\x94\x97\xf0\xce\x12\x55\xd2\x11\x97\x99\x04\x37\xce\xcb\x0c\x91\x5b\x52\x72\xa5\xca\x28\xf0\xa6\x59\x02\x7e\xa1\x02\x1d\xaa\xd9\x8e\x14\xc9\x24\x2f\xb3\x38\xfa\x13\xbc\xbd\xbe\x93\xaf\x23\xb4\xaf\x99\xd7\x0e\xe5\xbf\x04\xd5\x5e\x55\x74\x8e\x46\x1e\x2c\x05\x41\xbe\xbf\x9e\x54\x74\x1e\x99\xd1\x3d\x5f\xf0\xea\x80\x12\x26\x04\x51\xe0\xcd\xc9\x3d\x47\x98\x12\x8c\xaa\x12\x49\x2e\xaf\x28\x9a\x57\x0b\x46\xd2\x22\x4f\x6f\x14\xaa\xb2\xc5\x05\xb9\xe7\x50\xd7\x5b\x62\x34\xa3\x03\x77\x4c\x26\x8c\x70\x58\x3d\x12\xf9\xff\xd7\x24\x9f\xce\x38\x3a\x10\x25\x69\x91\x93\x52\x95\x0c\xa1\xcd\x63\xd1\x3e\x49\x19\x8b\x3b\x33\x28\xee\xf4\x50\x07\x2f\x78\xd5\x69\x96\x92\x22\x61\x29\xad\x8a\x42\x01\xdc\x57\x7d\xe9\xe5\xc0\xcc\xef\x7d\x4d\xc3\xf4\xe0\x0a\xfb\xcb\x12\xcf\xc9\x48\xd4\xbb\x8a\x1c\xbe\xb8\xaf\x69\x72\x43\x96\x35\x25\x8c\xc5\x76\x78\x7a\x74\x69\x55\x32\x8e\x88\x60\x01\x21\xc1\xdf\x49\xfc\x85\x00\x93\xe4\x6e\x96\xa7\x33\x34\x1a\xa9\xd7\x4f\x9e\xa0\x47\x24\x61\xb3\x7c\xc2\xff\x8d\x2c\x35\x80\xe6\xa4\x25\x6c\x31\x9e\xe7\x3c\xee\x0e\xd5\x6b\x92\xd4\x14\x18\xe5\xa5\x54\x65\xfa\xcd\x4a\x51\x0a\x16\xbf\xa4\x2a\xe3\xce\x0d\x59\x2e\x6a\x39\x5b\x9d\x1e\xca\xc8\xb8\x5a\x94\x29\x89\x5b\x6b\x27\x6a\xcc\x9b\x5d\x3f\x11\x5a\xf5\x42\xe6\x86\x14\x94\x55\xd7\xa7\x67\x02\xac\x10\x20\xca\x3a\xf0\x6d\x00\xb0\xe8\x4b\x91\x73\xcd\x00\xa7\x5e\x46\xb2\x45\xfd\x13\x2f\xb5\x24\x58\x42\x29\x76\x86\x0a\xd7\x63\x5e\xba\xb3\x56\xe2\x71\x41\x5e\x8a\x37\xeb\xda\x01\x99\xe4\x9c\x03\x04\x77\xd2\x6b\x4c\xc5\xaa\xf4\x91\xb0\xba\x2a\x19\xd9\xd4\xbb\xaa\x2a\xf4\x2b\xd4\x6d\x20\xd2\x80\xb4\x03\x32\x4d\x80\x2e\x5e\xb0\xe8\x9c\xb9\xcb\xd1\x06\x40\xce\x0a\xe5\xc2\x10\x7a\xf1\x86\x64\x9b\xc6\xa4\xaa\x34\x86\xa2\x4a\x77\xe8\x59\xd5\x74\x7b\xcd\x4b\x46\x28\x7f\x4b\x38\xcd\xd3\x75\x10\x18\x29\x48\xaa\x40\xc8\xfa\xd7\x73\x68\xe0\x91\x80\x4c\x28\x61\xb3\x33\x21\x51\xb7\xb8\xd8\x05\x96\x6a\xe2\x42\x99\xe3\xfb\x73\x58\x17\x3f\x9a\x65\x71\x23\x59\x5d\x70\xc1\x45\xd5\xa1\xb3\xe1\x5c\xa1\x1a\xaa\x82\x5c\xc0\xda\x1e\x52\xc4\xaa\x42\xd4\x58\xc4\x44\x03\xb4\xa6\x89\xd4\xfe\x66\x3d\x89\x1c\x41\xe1\x78\xcc\xc2\xad\xf0\xa5\x30\x78\x0f\x78\x35\x9d\x16\x64\xd4\xe1\x78\xdc\x71\x89\x21\x1a\x26\xe4\xbf\x5a\x76\x4b\x57\xfc\xc4\x11\x9b\x55\x77\xcd\xda\x55\x29\xcb\xcb\x64\x0c\x55\xa3\x1e\x6a\x6b\x01\xa1\xf8\x39\xa6\x53\x50\xfc\x8f\x63\x92\xc8\x07\xa5\x68\x02\xf6\x8f\x7c\x2f\x64\x86\x94\x3c\xee\x26\x79\x99\x91\xfb\xd8\xad\xef\xea\x08\xfd\x42\xe8\xda\xc7\x71\xf4\x27\xb1\x16\x2a\x08\x98\x73\x1a\x47\x98\xe6\xf8\x40\xdb\x33\x51\xb7\x9b\xcc\x30\x7b\x51\x60\xc6\xe2\x88\x92\xa2\xc2\x59\xd4\x6d\x28\x61\xa9\x7a\xc1\xea\x70\xb5\xec\xca\x2c\x93\x1f\x09\x5f\xd0\x12\x89\x4d\x07\x43\x93\x2a\x5d\x30\x34\xc6\xe9\x8d\xb0\x06\x60\x81\xc9\x4b\xc6\x09\xce\x50\x35\x41\x12\x96\x30\x0a\x92\x90\x10\x24\x63\x98\x9a\x1b\xb2\xcc\xaa\xbb\x52\x98\xd3\x14\x60\x07\x29\x69\x15\x26\xf4\xe9\x91\x04\x8a\x6f\x71\x11\xfb\x4f\x5d\x55\x47\x42\x5d\xb3\x88\xac\xba\x56\x1d\x53\x5a\xad\x59\x20\xe5\xbb\xa8\x9b\xcc\xf2\x4c\x51\x1d\x9a\xdc\x61\x5a\x0a\x9b\x27\xdc\x48\xbd\x6d\x37\x83\xca\xcf\xa5\xb9\xb0\x9e\xc5\xc5\xc2\xd1\x14\x0c\x2d\x9d\x06\x82\xd7\xc4\xa9\xbd\x7c\x7e\x9f\xb3\xb5\xb5\x97\xd7\xf8\x3e\x67\x4e\xf5\x82\x4c\x49\x99\xad\x41\x47\xbe\x74\xf5\x60\x9d\x97\x25\x59\x47\x2b\xf5\xd6\x5d\x8b\x6e\x71\x71\xce\x31\x5f\x23\x9c\xf0\xfe\x9a\x89\x0a\xae\x34\x93\x32\x7b\x89\x39\x09\xb7\x71\x74\x2d\x29\xb3\xb6\x8e\x57\x8d\xc5\x3e\x97\x88\x5d\x6b\x9d\xa7\x37\x84\xc6\x92\x99\x8a\x2a\xc5\x05\x19\xa0\x0e\x29\x3b\xd2\x18\x17\xa6\x20\xe6\x03\xd4\xf9\xed\xb7\xdf\x7e\x3b\x78\xfb\xf6\xe0\xe5\x4b\xf4\xfa\xf5\x60\x3e\x57\xef\x79\x55\x15\x63\x4c\x3f\x14\x38\x05\xeb\x76\x80\x3a\xe3\x8a\xf3\x4a\xbf\x67\x79\x46\x7e\x5a\x9e\xe7\x19\x19\x20\x4e\x17\x44\x95\xce\xaa\xbb\x8b\x2a\xc3\xcb\x9f\x16\x9c\x57\x65\xf3\xd5\x8b\x82\x60\xda\x2e\xac\x98\x07\x44\x60\xff\x1f\x55\x29\xd0\xfd\x74\xf1\x02\xfa\x5b\x75\x83\x1b\x2d\x43\x08\x5f\x68\x2c\x25\x70\xdc\x11\xff\xbd\xc8\xe7\xe4\x03\xd0\xa3\xd3\x05\x02\xad\x03\xa3\x37\x63\x1e\x1c\xa1\xf8\xb2\x5a\xd9\x2d\xae\xac\x76\xd1\x97\x90\x0e\x51\xd8\x86\x96\x2e\x6d\xfc\xb4\x41\x2c\x6a\x81\xd7\x47\x59\x5d\x03\x31\x4a\x84\x9d\x9b\x85\xb8\x65\xd9\x29\x69\x77\xd7\x6b\xa9\x0d\x60\xaf\xd8\x39\xea\x34\xac\xe2\x79\x25\xe6\x73\x2b\x93\xc9\x6a\x6d\x3e\x93\xe5\x7f\x98\xcd\x06\x8c\xfd\xdf\xc4\x69\xa2\x26\xe3\x78\x5e\xbb\x0b\x5d\x26\x85\xb5\x24\x77\xe8\x65\x8b\xa9\x4c\x8b\xa7\x47\x87\x87\x87\x5d\xcb\x9e\x96\x80\x6b\xb9\x53\xfc\x48\x5e\x44\xa4\x60\x24\xe0\x1b\x70\x26\xc7\xe3\xfd\x1d\x80\xaf\x07\xe4\x71\xbf\x82\xf4\x4d\xcc\xaf\x1d\x39\x7c\x59\x10\xe0\x5c\x69\x79\xb6\x58\x57\x54\xca\xd3\xca\x58\xa5\xd6\x4e\x95\xfc\xd8\x49\xa6\xc5\xb2\x9e\x89\x2a\x1d\x67\xe5\xf7\x65\x22\x6e\xad\xe8\x16\x0a\xce\x32\xb5\xfa\x8f\x79\x79\x50\xd3\x7c\x8e\xe9\x32\x32\xdb\x2c\x01\xd8\xa9\x63\x3a\x3b\x48\x67\x24\xbd\x69\xd4\xa3\xe0\x77\x6c\x55\x5d\x94\x50\x99\x64\xba\xba\x9a\xb3\x75\x28\x79\x60\x1e\x86\x55\xab\xab\xcd\x98\x79\x83\x58\x69\x07\x8b\x37\x29\xb1\xa3\x64\x1c\x1c\x1b\x7b\x3c\x4d\xdf\x10\xed\xc5\x0e\xd7\x2e\xb9\xff\x7a\xfe\xfe\x9d\x9d\x8d\x7e\x1f\x9d\x4d\x1c\x97\xc8\x1d\x66\x48\xf5\xd2\x83\xe2\x8a\xe6\xd3\xbc\xc4\x05\x62\x84\xe6\x84\x21\xf0\xd1\x4e\x2b\x8e\xe6\x0b\x8e\x39\xc9\x2c\x9c\x98\x09\xcd\x92\x75\xc1\x45\x75\x47\x50\x49\x48\x26\x2c\x30\x4a\x60\x33\x4e\x17\x29\x47\x39\x97\x2e\x2b\x0f\xb2\xc0\x08\xe0\x26\xee\x7c\x28\x67\xb0\x34\x6e\x29\x2e\x99\xd0\x53\x2f\x85\xd0\x34\xc6\xe2\x6e\xbb\x5b\x1a\xb6\x45\x8b\x1f\x51\xe7\xb0\x83\x06\x42\xe9\x6a\x73\xad\x49\x6d\x03\x48\x2a\x7c\x70\x5f\xc6\xee\x46\xb9\xdf\x47\x17\x60\xb4\x1b\x71\xb1\x0e\x4c\x69\xcd\xbf\xf7\x64\x49\x9a\xbf\xa7\x45\x0f\xe5\x9c\xcc\x35\xf1\x1f\x9b\xf2\xae\x65\xa5\x4e\x83\x37\x3a\x5d\x8f\x75\x3a\x01\x4e\xd3\xe3\x00\xcd\x7e\xce\x2b\x8a\xa7\x24\x61\x84\x9f\x71\x32\x8f\x45\x87\x3d\x33\xd8\xd5\xb0\x81\xe6\x64\xf2\xc7\xf1\x74\x10\x59\x87\xab\x1a\xca\x76\x3c\x0f\x3b\x8e\x83\xa8\xdf\x47\xe0\x2a\x28\xf2\x14\x0b\xfc\x04\x8d\x0b\xf0\x1d\x67\x8b\xfa\xda\x55\x4a\xda\x19\xb1\x46\x25\x29\x26\x00\x60\xa7\xe0\x82\x08\xae\xd2\x02\xf6\x2d\x1a\xf9\x08\x4e\x15\x82\x1d\xe9\xbb\x38\x80\xae\x3a\x92\x0f\xb4\xf8\x94\x15\x47\x8c\x70\x94\x97\xb2\x2d\x93\x6d\x7b\x68\x8e\x6f\x88\xe0\x79\xd9\x36\x4b\x5c\x63\xe0\x56\xaf\xfd\xe8\xeb\x57\xf5\xa0\x0f\x43\xe4\xf0\x1d\x51\x76\x31\xb7\xba\xd4\xe3\xb7\xd8\x52\xa5\x87\x5a\xc8\x36\x54\x9e\xcf\x02\x5b\x9b\xb6\x9c\x3e\xeb\xb4\x4f\xd3\xc9\xe3\x8a\xa0\x3f\x86\x80\x1c\x8a\xf1\x5a\x4c\xec\xae\x30\xcc\x6a\x8d\x05\x65\x57\x32\x78\xeb\xf5\x96\xde\x14\xd7\x86\x7b\xda\x4e\xb6\xe6\x26\xf5\x83\xf4\x20\x21\xed\x41\xd2\xec\xdc\x72\x55\xb9\x9c\xdd\x76\x79\x6d\xe6\xf1\x0f\x7e\xfd\x3f\xca\xed\xaa\xfb\x03\x8d\xdb\x3f\x9b\xf1\xc3\xc3\x59\x27\x02\x41\x4a\xda\xb9\x09\x0d\x66\xb3\x60\x7c\x1b\xc0\xbd\xf5\xfe\xca\x8d\x82\xd3\xa8\xee\x0a\xcf\x3a\x3a\xac\x11\xa3\x20\xde\xdf\x2a\x51\xdf\x44\x04\x4f\xce\x76\x47\x67\x9b\xc8\x7d\x2b\x2e\x0d\x27\xf7\x3a\x3f\xa3\x25\xf9\xc6\xa3\x3a\x07\x52\xeb\x7c\x20\x3c\xb5\xd6\xe7\xe8\x58\x8d\xda\xdb\xe5\x58\x1c\xda\xd1\xb8\xb9\x56\xc8\xdd\xb6\xce\x51\xa6\x84\x6e\x82\x0b\x46\x86\x66\xf3\xe4\x7a\x4c\x8c\x23\xa8\x3d\x26\xb9\xe3\x1c\xc3\xf6\x4d\x3b\x80\xd3\x6b\xf0\x60\x5f\x45\xdd\x00\x33\x6b\x87\x5a\x4a\x09\x66\xe4\xa3\x42\xd0\xed\x74\x13\xf0\x8c\xec\x00\x3c\x23\x01\xe0\xbb\xa2\x4e\xca\x6c\x17\xc4\x4f\xcb\xec\x81\x68\x6f\x01\xac\x91\x76\x00\xef\x8a\xb2\xdc\xe4\xed\x82\xf5\x5b\xa8\xf9\x40\xc4\xb7\x83\xd7\xb8\xfb\xe0\x83\xce\xd3\x80\xeb\xa4\xe1\x11\x95\x1e\x7b\xf1\x2e\xa2\xa4\x2e\x70\x2a\x36\xac\x5f\x38\xb9\xe7\x83\x00\x3c\xf0\x91\xf4\xd0\xbc\xca\xc8\x00\x45\x63\x32\xa9\x28\x89\x56\x2d\x37\xab\xf6\xbe\x8a\x55\x89\x12\x78\xca\xcb\xa9\x95\x79\x79\x24\x2a\x2c\x53\xb9\xc9\x0d\x78\x55\xf4\x91\x84\xa8\xa4\x5c\x29\xa6\xc5\x46\xa5\x2d\x6b\x6d\x92\x36\x73\xee\x20\x74\x9b\xd8\xeb\xbf\xa4\xf9\x44\x79\x7a\xfb\x7d\xe4\x84\x5a\xc0\x5c\xa1\x59\x2e\x96\xd1\xa5\x5a\x08\x1f\x05\xd7\xe7\x48\x55\xb2\xae\xf2\xa0\x55\x6d\xaa\xf5\x60\x93\x95\x30\x4e\xf3\x72\x9a\x4f\x96\xf1\xe5\x55\xd7\xf7\x25\xd4\x55\xbd\x28\x30\x27\x67\x40\x7f\xa1\x4d\xe5\x1c\x30\xa5\x19\xcc\xa2\xec\xb8\xb2\x5d\x3a\xb4\x54\xcf\x2a\x1c\x1b\x63\x63\x4c\x7c\x7a\xac\xf3\x3d\x30\x3f\xd2\x44\x16\x8e\x69\x75\xc7\x08\x15\x8d\x5d\xe7\x4e\x57\xd0\x47\x14\xc6\x5d\xd4\x57\x01\x57\xb0\x83\x49\xf0\x67\x7c\x1f\xeb\x35\x05\x21\x81\x52\x95\x0d\x50\xf4\xd7\xd3\x8b\xa8\x67\x8a\x17\xb4\xf0\x62\x22\xd0\x3e\x8a\xfa\xb8\xce\xfb\xb7\x47\x7d\x98\x9b\x1f\xe1\x77\xc4\xa1\x0b\xa7\xa1\xd8\xab\x5e\x2c\x6b\xc1\xa4\x9f\x59\x55\x3a\x6f\x80\x3e\x8b\x34\x25\x8c\x0d\xec\x00\x45\xa5\x1e\x1c\xe6\x9f\x73\xcc\x17\xcc\xae\x77\xca\x65\x30\x41\x50\x47\x6c\x65\xf9\x82\xa1\x47\xa3\x11\x8a\x14\x98\xa8\x59\xd9\x4e\xc1\xac\xba\x3b\xa5\xb4\xa2\x71\x04\x7f\x24\x3f\xe5\xe5\x14\x9c\x68\x89\xf5\x48\xd8\x7f\x92\x5f\xfd\xf2\x95\xf7\x24\xe7\x80\xde\x1a\x6a\x03\x5e\xb0\x5b\xa7\x84\x2d\x0a\x7e\x79\x78\x35\x6c\xb5\xc8\x72\xd8\x59\xbe\xc5\x7c\x96\xe0\x31\x8b\xdd\x09\x3b\x70\xe0\x69\x73\xd2\x1d\x38\xb4\x3d\x19\xa1\xef\x0e\xdb\x23\x85\x98\x30\x31\xce\x5f\xe4\x31\x46\xdc\x1a\x11\x42\xd1\x71\x96\xdf\xa2\x54\xac\x9e\xa3\xdf\x23\x5c\x10\xca\x11\xfc\x1e\xa8\xb3\x8f\xdf\xa3\x93\x63\xc6\x69\x55\x4e\x4f\x14\x98\x47\xc7\x7d\x55\x80\x5e\x12\x4e\x52\x4e\x32\x14\xa1\xfd\x00\x70\x81\x5c\xc2\xab\x57\xf9\x3d\xc9\xe2\x67\xdd\x60\x9d\x08\x31\x92\x56\x65\xc6\x80\xee\xd0\x44\x86\xa7\xa0\x31\xe1\x77\x84\x94\x68\x59\x2d\x0c\x13\x83\xbb\x84\xcf\x88\xa2\x4a\xe2\x06\x20\x52\x52\xe4\x84\xa1\xaa\x44\x38\x4d\x17\x14\x73\x22\x41\x42\x13\x80\x0d\xa2\x33\x87\x10\x8b\x14\x2f\x18\x41\x8b\x92\xdc\xd7\x72\x04\x52\x9d\xc8\x59\x62\xc9\x71\x3f\xcb\x6f\x4f\xa2\x06\xbe\xdd\x75\x73\xbf\xb2\x3c\x0c\xe7\x4c\x83\xb6\x78\xea\x7f\x61\xe6\x13\x56\x4b\x90\xf7\x64\x1f\xab\x75\xe1\x7c\x56\x41\xac\x55\x49\x3b\xc5\xa4\x35\x84\x3e\x28\xf2\x9b\x04\xbe\xc0\x63\x52\xf4\xaf\xaf\xc5\xc2\x70\x7d\xdd\xbf\x85\x78\x3e\xd3\x72\x9d\xc4\x3f\x4c\xd6\x1f\x20\xe7\x9b\x89\x8c\x6f\x71\x5e\x08\x0a\x21\x79\x34\xcf\x1e\xf9\xd2\xde\x94\xf3\x95\x15\xbb\x1a\x4f\xc9\x8b\xaa\x9c\xe4\xd3\x04\x17\x85\xa5\xb0\x91\x73\x58\x56\x79\x95\x55\x03\x94\x55\xc6\xb1\x07\xf8\xd8\x06\x3f\xa2\xf7\x14\xa5\xb8\x14\x9b\xbe\xcf\x0b\xc6\x51\x91\xdf\x12\xc1\xb8\x82\xb3\x45\x17\xa6\xbf\x49\x45\x51\x0c\x5e\x65\x08\x43\x44\x39\x3a\x0e\xe3\x90\x14\xa4\x9c\xf2\xd9\x10\xe5\xfb\xfb\x01\x5a\xb8\x86\xc2\xe5\xe1\x95\xb1\xd8\x71\x96\xc5\x62\x45\x78\x0f\xcf\x71\x10\xf4\x65\x7e\xd5\x0b\x77\x7a\x99\x5f\x75\xbb\x41\x3a\x41\xa7\x93\xc5\xdf\xfe\xb6\xfc\x08\x12\x65\x42\xea\xe4\x3f\x10\xb6\x01\xec\x64\x7b\x1e\xe1\x45\xdd\x76\xf9\x1c\xd7\x03\xf4\x65\xb5\xb6\x23\x61\x15\x08\xfe\xc2\x33\x82\x65\xec\x9b\xc1\xca\x48\xe6\x26\xb9\xfc\x76\x76\x59\xe9\x23\x96\x2d\xd2\xe9\x61\xe8\x4a\x24\x20\x0b\xa8\xc8\x20\x2c\xb9\x7d\x42\x23\x49\xa2\xd7\xd2\x22\x49\x72\xe6\x6e\x62\x9d\xb9\x30\xb5\x34\x1b\xa4\x55\x99\x62\x1e\x9e\xc8\x2e\x1a\x84\xe7\xd1\x0f\x14\xe3\x86\x92\x92\x42\x78\xc1\xab\x73\xb0\x44\x07\xd2\x56\x53\x07\x50\x80\xe9\x40\xfd\x95\x65\x39\x27\x73\x36\x00\x63\x42\x16\xcc\x31\x4f\x67\xc4\xa1\x3b\x8a\x5d\x6f\xa9\x76\x8d\xdc\x11\x34\xc3\xb7\x44\x31\x00\x70\x7d\xba\xa0\x94\x94\x5c\xd2\xa1\x87\xd8\x4d\x5e\xef\x59\x3d\xd0\xe4\x2f\x49\x08\x50\x09\xb0\xea\xc1\x63\x6b\x8a\xdb\x0d\xdc\xea\xc3\xf5\x95\xe7\xb8\x16\x1c\xbc\xda\x50\x85\x6a\x3e\x87\xc2\x64\x92\x17\x9c\xd0\xd8\x42\x4f\x94\x05\x1f\xf7\x51\x7f\xda\x43\x51\xd4\xed\xa9\x05\x5a\xd2\xcf\x93\x8f\x9a\x0a\x5d\xa9\xd7\x5d\xcf\x42\xaa\x2b\xc6\xc5\x3b\xbd\x06\xdb\x35\x6a\xd5\xdd\x8a\x5e\x32\xa9\xe8\x29\x4e\x67\xd6\x3c\xa7\x01\x65\xd1\x18\xf9\x25\x4d\xf4\xe9\xc3\x15\x1a\x21\x3a\x0c\xf4\x68\x24\x52\xd9\xf4\x62\x92\x51\x5e\x06\xe1\xe9\x18\xbd\x3d\xc5\x46\x94\xb7\x19\xc4\x51\xfc\xf0\x98\x88\x6a\x16\x6b\xdc\x1b\xbb\x78\x6b\x05\x19\xc4\x7e\x7c\x95\xb0\xb4\xa2\xd2\x94\x0a\xbc\xc7\xea\xbd\x1d\x96\x1e\x03\x78\xdb\x0e\xd1\x8f\x08\x27\xf2\x20\xf8\x45\x35\xaf\x31\x25\xf1\x58\x48\x52\x6e\xc6\x6e\xa8\xe0\x0c\x9e\xf9\xbe\x14\x79\x0c\x32\xcb\x19\xac\x07\x10\x79\x3b\x83\x50\x5d\x84\x27\x5c\x98\x35\x9c\xe3\x74\x06\x16\xc0\x8c\x20\x23\x81\xa8\x2e\x16\xd3\xbc\xec\x21\xcc\x50\xce\x25\x94\x8a\xcf\x08\xbd\xcb\x19\x41\x63\x4a\xf0\x0d\x6b\xb4\xd0\x34\xc2\x45\xce\x97\xc9\xde\x9a\xb8\x1b\x4f\xbb\x8c\xf3\x32\x53\xff\x3f\xbd\x25\x25\x67\x5a\x85\xae\x36\xea\xb4\x29\xe1\xef\x4d\xc0\xf4\x76\x13\xa3\x11\x60\xbd\x1a\xfa\x51\xd7\xe0\x52\xd2\x57\x00\x10\x8a\x9c\xc8\x3f\xc5\xff\x91\x09\x6b\xd0\x05\x8c\x93\xba\x59\x02\x47\x57\xfa\xd1\x3d\x0d\x16\x82\x72\xb5\xde\x93\x20\xeb\x74\x13\xe2\x89\x07\x44\x6f\xf5\x74\x44\xb4\xbb\xd5\x12\x96\x8e\xbd\x49\x92\x88\x47\x27\x94\x2b\xc9\xcb\xe7\x94\xe2\x65\x2c\xca\x7b\xde\x10\xbb\xc2\x5c\x77\xac\x75\x08\xb3\x55\x50\xc0\x6e\x52\x4b\x39\x3a\x41\x9e\x4d\xaf\x68\x07\x7b\xef\x2b\xa7\x67\x68\x63\xe5\x10\xf9\x1e\xbc\x6d\x51\xf4\xdb\x3d\x7c\x3e\x1c\x15\x82\xdc\xd8\xdc\x0e\x9d\x1a\x32\x0c\x8e\x35\x22\xe3\xa4\x8f\x00\xc4\xc3\xdc\xa4\xd9\x66\xd1\x62\xca\xc8\x4b\x61\xc8\x4b\x54\xad\xce\x12\xac\x71\x41\xee\xb9\xe5\x35\x28\xfa\x78\xaa\xf6\xb7\x1f\xc9\xf4\xf4\xbe\x8e\xa3\xff\x8c\x2f\x0f\x0f\xfe\x72\xb5\xdf\x8d\x2f\x97\x77\xd9\x6c\xce\xae\xf6\xbb\x8f\xe5\xe2\x2d\x1a\xc9\xc5\x49\xf0\x9c\x81\x98\x40\x59\xac\xc0\x99\xd0\x89\x47\xaa\x6a\x17\x7d\xd1\xd6\xa1\xb9\xe3\xa0\x5e\xe9\x59\x7b\x34\x42\xdf\x35\x22\x59\x7e\x38\xd4\xce\x03\xd1\x2b\xcc\x17\x1a\x21\x18\xde\x59\xc9\x35\x80\xcb\xa3\x2b\x83\xd9\xa2\xcc\xc5\x52\xa2\xdf\x3c\xbb\x72\xc8\x27\xdb\x3f\x6d\x5f\x1e\x71\xae\xf6\x5c\x0a\x00\x57\x3b\x58\x25\x8e\x77\x70\x67\x21\x06\xe2\x9c\xab\x4d\x9b\x3d\x79\xb1\x73\x25\x57\x67\x1b\xe7\xeb\xc4\xf2\x85\xec\xd9\x0d\x37\x82\x42\x56\xad\xa0\xb9\x87\xc2\x71\x08\x85\x0d\x40\xc1\x6a\xf5\x7d\xe6\x0d\x5c\xb7\x34\x1e\x36\x0d\x91\xb6\x97\x07\x6d\x70\x32\xdb\x8d\xa3\xbb\xd1\x58\xed\xe2\x05\xf2\xdc\xb9\xff\xfc\x09\xdb\x3e\x53\xe8\x00\x1d\x89\x59\x3d\x91\xb3\x7b\x70\xb0\x76\xd6\x4e\xfe\xff\x99\xb5\x29\xe1\xa7\x26\x12\x72\xfb\x94\x81\xc2\x71\x03\xf6\xd0\xd7\xaf\xc8\x2b\xf0\xb1\xa6\x3a\x9e\x57\x79\x9c\x95\xae\x71\xc3\xe8\x76\x89\x20\xdc\xbe\x89\x11\x0b\x3e\x3d\x7f\xd8\x60\x9c\xb0\x32\x79\x66\x63\x9a\x3b\xd1\xb4\xcc\x16\x9a\x48\x31\x85\x7e\x06\x37\x48\xb7\x20\xc6\x82\x38\x01\xa8\x8d\x97\xf0\x76\x21\x8b\x42\x68\x47\x4d\x7a\x5a\x06\x8e\x8f\xd7\x90\xa5\x24\x77\x0a\x65\x35\x75\x9a\x40\x2e\x91\x95\x18\xaa\xba\xb0\x5f\xdf\x59\x7e\x51\x1f\x3d\xeb\xa1\x8e\xf2\xaf\x75\x82\xf4\x56\x80\x9d\x77\x3e\xeb\xef\xa8\x90\xfe\xd1\xe3\x66\x8b\x31\xa7\x38\xe5\xff\x47\x0d\x7e\x4a\xf8\x5b\x1d\x7b\xfa\x10\xb1\x56\x01\xab\x46\xaa\x55\x64\xe2\x43\x85\x7a\x87\xd0\xc8\xdd\x65\xfa\x01\x03\x09\x88\xf4\x5b\x07\x4d\x4d\x64\x55\xf6\xad\x02\xdd\x46\x68\xbb\x3c\xef\x1e\x89\xba\xa3\x38\x3f\x90\x2a\x9b\x39\x5b\x13\xa9\x25\xd0\x47\x87\xeb\x18\x55\x35\xf9\x3b\x09\xe9\x3f\x7e\x34\x46\x4c\xff\xd1\x43\x72\x6a\xef\x7e\xe7\x3a\x2d\x08\xa6\xd2\xc5\xd7\xf5\x0b\xf5\x01\x49\xb7\xb1\xfe\xb6\x2c\x04\xbb\xf6\xdb\xc0\x15\x1d\x19\xc0\x66\xd5\x5d\x1c\xb8\x32\x91\x90\x79\xcd\x97\xb1\x1b\x46\x8c\x29\xdf\x70\x1c\xf7\xf7\xb0\xdb\xd4\xc5\x56\xbb\xd1\x33\xdb\x8d\xf5\xbb\x5f\x7d\xf1\x4d\x6f\xaa\xaf\xa2\xae\x1e\xfd\xd7\xaf\xf2\x78\x6a\x8e\xef\x63\xf8\xcf\xa4\xa8\x2a\xea\x5b\x74\x7d\xf4\xec\xfb\xc3\x6e\x0f\x1d\x39\x1b\xac\xd6\xbe\x72\xb7\x2d\xa7\x6a\x6f\xef\x8f\xb4\x6c\x07\xe7\xc8\x12\x7a\xd2\xac\xdd\xd4\x48\xa6\x9e\x7b\x2a\x0b\xa3\xff\x75\x46\xbd\x33\x59\x5d\x98\xe0\x71\x45\x1d\x95\x0b\xfb\x31\x5a\xe8\x9e\xd4\x81\x83\x7e\xac\x31\xc5\x73\x7b\x63\x3c\x02\x28\xd1\xa0\xb9\x41\x36\xe1\x5a\xb2\xbe\x8c\x0d\x44\xa3\x35\x31\x8b\xe8\x47\xd4\xe1\x74\x41\x20\x56\x09\x5c\xae\x52\x86\x54\xe3\x66\x5c\x8f\x03\x67\x53\xe8\x53\x1b\xe2\xa6\xbb\xfe\xc6\xf7\xa0\x3a\x05\xa6\x45\x23\x33\x27\x07\x1e\x7b\x0e\xdd\xaa\xf2\x1e\x92\xaa\x38\xf4\x81\x10\x31\x6a\xcb\x98\xde\xdb\x75\x3e\x8a\x00\xaf\xc8\x76\x0b\x5a\x88\x5d\xc9\x86\x33\x68\x19\x21\x13\xa9\x18\x08\x39\x75\xae\xc2\x08\x9c\x37\xb9\x51\x43\xa0\x76\x34\x31\xdb\x95\x87\x32\x46\xd8\x0b\x8a\x53\x63\xe1\x52\xbc\x25\x5f\xee\x8a\xed\x37\xe3\xf9\x42\x46\x4c\x6d\xc7\xd4\x09\xb6\x93\x6c\x2b\xff\xd3\x70\x8b\xfd\x3a\xa3\x68\xb4\xee\x4c\xb0\xa1\x3f\xe4\x35\x45\xf9\x32\xea\x7a\x67\x85\x0b\x5a\x6c\x3b\x01\x14\xe5\x03\x85\xc4\x3f\xfb\x54\x10\x5a\xc1\xb1\xd0\x96\xd3\xbf\x56\x57\xea\x68\x9c\xad\x01\xaf\x57\x13\xbf\x6e\xf0\x9c\xcc\xf3\xc3\xca\xb9\x54\x4f\xfe\x31\x96\xa1\x4a\x6c\x4e\x1a\xfd\xf9\xdd\x76\xdc\x75\x3f\xa3\x3d\x08\xa6\x6d\xd2\x4e\x94\xa1\x47\x23\x14\x81\xda\x6b\x50\x0c\x94\x30\xa5\x2e\x79\x44\x9b\xfb\x19\x4d\xb4\xf2\x81\xfb\x08\x8f\x42\xc9\x41\xf4\x3f\x42\x05\x37\x35\xdb\x48\xca\xbb\x90\x1b\xf7\x4c\x9a\x8d\xe5\xfc\x5e\x90\x7b\xee\x35\xda\x7a\xea\x4b\xee\x49\xba\x80\xbc\x16\xea\xd4\x31\x42\xfb\x02\x6c\xeb\x90\xdd\xa1\x5e\x5a\xcd\xeb\x82\x70\xb2\x33\x01\x47\x6b\x08\xb8\x9e\x99\xc0\x8a\xb6\xce\xcd\x60\x58\xce\x81\x35\x15\x86\x5e\x43\x5e\x71\x5c\x88\xe2\x73\x79\xcf\x04\x52\xd4\x6c\x9a\x21\x79\x41\x64\xc3\x34\xad\x6d\xa4\x4e\x8e\x84\xf0\xc2\xba\x10\xb1\x14\x17\x98\xb6\x02\x6b\xda\x28\x1d\x05\x26\x37\x9f\x6c\xec\x05\x30\x2c\x17\x45\xb1\x1d\xfa\x26\x30\xda\x6d\x18\xe4\x93\x95\xef\xea\xb1\x66\xda\x8c\xcf\x8b\x38\x7a\x53\x61\x19\x2e\x22\x19\xc5\x4c\xd1\x3e\x8a\xe6\x0c\x1d\x8f\x29\xea\x9f\x20\xbb\x10\xc9\x5a\xce\x72\xb5\x8f\x22\x5d\x4d\xbc\x89\x2e\x04\xe6\x32\xfe\x44\x5e\x0a\x92\x2d\x1a\x03\x6a\x1e\xdd\x35\xa3\x4d\x2d\xea\x3b\x1c\x36\x1b\x11\x70\x57\x90\x39\x9b\x6e\x71\x86\x88\x16\x89\xd0\x29\x50\xb7\x51\xae\x8d\xdb\x6d\x61\x6a\xc6\xc4\xde\xdd\x2c\x77\x3a\xee\x74\x9a\xfd\x6a\x02\xec\x30\xe4\x5f\xcc\x55\xee\xdd\x07\xad\xb4\xb3\x9c\x7b\x6f\xd8\xfa\xcd\x43\x06\x1e\xc0\xe0\x01\xdd\xbb\x83\x37\x2f\x76\x1b\xbe\x77\xd9\x76\x87\xee\x5d\xc3\x4f\xb0\x66\xb5\xe0\x67\x2f\xb5\xcc\xdd\xe5\x65\x56\xdd\xc9\x11\x5d\xc8\x97\xcd\x9a\x66\x03\x94\x37\x52\x58\x84\xb6\x27\x8d\x1b\xc3\x76\x8f\x02\x1b\x2d\x0d\xc1\x3f\x5a\x31\x37\xaf\x74\x97\x68\xa4\xf1\x62\x52\x3d\x0a\xac\xc2\x41\xa6\x01\xe7\x2d\x94\x37\x6f\x24\xef\x41\xa2\x18\x33\x82\xa7\x2a\xfb\xdc\x76\x6a\xcb\x74\x4c\x6f\xf0\x98\x14\x9e\x91\x06\x41\x50\xcc\x92\x1c\x9e\xcf\x21\x8a\x94\xa9\x4c\x6d\x8e\x43\x1d\xde\xc2\x65\x10\xa7\x99\x24\x8a\x7c\x25\x16\x65\x1d\x51\xe5\xa8\x5b\x17\x6a\x52\x2f\xd8\x2c\xb6\x81\x02\x68\x5f\x81\xdd\x77\x22\x04\xd4\x8a\xc7\x52\x5c\x93\xd7\x17\x6f\xdf\x28\x3c\x2f\xe1\x8f\x09\xe0\x59\xf9\x1e\xa6\x42\x8f\xce\x0f\x10\x94\xc5\xbf\x47\xb6\x2b\x8d\xc9\xe7\x2a\x2f\xe3\xe8\x78\x4c\x4f\xa2\xae\xec\x1e\x22\xe8\xb6\x12\x53\xc6\xd4\x5c\x54\x17\xec\x9d\x3c\x51\x5d\x4b\x4e\xae\x6b\xa8\x37\x89\x26\x8e\xd8\x9d\x76\x3a\xd0\xeb\x97\x68\xb8\x89\xf8\x5b\xa9\xbf\x9d\xfc\x01\xfa\x1b\x92\x8f\x7e\x8f\x0c\x5d\x34\x7d\x45\xf9\xef\x91\x89\x18\x82\xd5\x47\xfc\xa8\xd1\xec\x8f\x42\x64\xec\x49\x1a\xae\x22\xc7\x71\x26\x1b\xec\x76\x6a\xfa\xb3\x3a\x63\x34\xb4\x84\x43\x43\x4b\x4a\x29\xb1\x50\xf5\x55\x51\x61\xae\xde\x6b\xa1\xcc\xd9\x3b\xfc\x4e\x94\x19\xbf\x47\xbf\x8f\xa2\xfd\xb3\x72\x12\xf5\x50\x74\xa0\xfe\xc2\x33\xba\xcb\x8b\x02\x8d\x89\x04\x96\x09\x71\xaa\xd0\x3b\xfc\x0e\x8d\x97\x2e\xfc\x6e\x82\x2e\x66\x44\x83\x4a\x71\xd9\xe1\xa2\x11\xc4\x96\x93\xac\x87\x58\x05\x17\xda\x11\x9f\x91\x39\xc2\x0c\x4d\x71\xcd\x50\x0c\x96\x80\x77\x1d\xca\xdc\x7d\xf2\xce\x43\xb7\x12\xc5\xbb\x12\xdb\xdc\x57\x6d\x74\x82\xd5\xb8\x20\xdc\xdc\x8a\xff\xa8\x12\x32\x26\x2f\xaa\xa2\xa2\xc9\x07\xf9\xd2\xba\x8d\xc0\x38\x77\x0c\x26\xc1\x43\x73\xcc\x69\x7e\x1f\xf9\x2a\xca\x1a\xa9\x2a\x3e\x2e\x67\x70\x3f\xac\x9a\x20\x59\x1f\x22\x3a\x1e\xa1\x0f\x05\xc1\x8c\xa8\xdc\x5b\x18\xa5\x15\xa5\x24\xe5\x90\xa7\x85\x30\x96\x57\xa5\x09\x16\x55\xd4\x90\x7c\xbe\xb2\x6e\x5a\xac\xa3\x13\xa9\x09\x79\xb1\x7a\x93\xb3\x66\x48\xc3\xd0\x3c\x49\x2e\xb6\x31\x0d\x9c\x29\x59\x35\x97\xd0\xac\x50\xa8\x60\x08\x6d\x1b\x0e\x5d\x55\xc5\x9c\xc8\xa9\x86\x89\xaf\x63\x28\xac\x6a\x92\xbe\x21\x4f\x25\xd8\x8e\x6d\xac\xa1\x01\x6c\xde\x59\x25\x66\x48\xe1\xf6\x32\x80\xdf\x9e\xd7\x7c\xa0\xfe\xfa\x7b\x51\xce\x64\x44\x05\xf3\x29\xe5\x08\x90\xfc\xd7\xe8\x44\xfc\xbb\x1f\xc8\xc3\xf9\xcb\xc3\x2b\x37\x62\x6b\x39\x70\xd6\x46\x90\x4c\x09\xed\xf2\xe8\xaa\x6b\xad\x52\x1b\x4d\x64\x37\x21\x85\xd8\xc2\x29\x0e\x4c\xe0\x31\x96\x2d\xe4\x66\x1e\xc8\x01\x66\x6f\x2b\xac\x8b\x39\x82\x2b\x23\x82\x61\xc6\x18\x28\x40\x5c\x14\x68\x9e\x33\x26\x4c\x15\xc6\x49\xcd\x12\xcb\x02\xe4\xce\x58\xd8\x4a\x65\x4a\x31\xa8\x9c\x4d\x86\x51\xa2\xdc\x59\xf6\x8d\x8f\x68\x88\x38\x3a\xf6\xcb\x49\x99\x89\xd2\xfd\x66\x6d\x52\x7b\x81\x80\xcf\x8b\xa2\xba\x03\xe8\x13\xa1\x34\x04\x7a\x75\x95\x97\x1c\xe5\xa5\x8c\xe8\x4e\x97\x89\x7b\x88\x2b\x4d\x7e\x13\x2d\x23\x70\x7c\xf2\x04\xc9\xe2\xcb\xba\x62\x57\xc9\x3d\x3a\x16\xfd\xb6\xba\x95\x5e\x41\x77\x3a\xcd\xc0\xa5\x4a\x77\x80\x38\xa6\x79\x5d\x41\x62\x4e\x35\x51\xcd\xed\x6a\x03\xc4\x97\xfb\x01\xe2\x3d\xa4\xc2\x5c\x57\xdd\x76\x88\x0e\x42\x26\x8b\xab\x69\x6b\x27\xd6\x9e\x97\xe0\x1d\xed\xbf\x56\x8a\xdc\x8d\x67\x51\x40\x13\x97\x82\xda\xeb\xe7\x9b\x61\x90\x30\x0a\x12\xd8\xe2\x72\x89\x38\xc5\x29\x61\x42\x4d\xe1\x12\x91\xfb\x5c\x26\x8c\x04\x35\x9e\xf8\x99\x88\xac\xd7\xdb\xe9\xce\xa6\x31\x4a\x67\x79\x91\x51\x52\xc6\xdd\x40\xb8\x93\xad\xdb\xb8\x31\x04\x2f\x20\x31\x92\xf7\x62\xd5\xcc\xb0\xf4\x38\xee\x38\x66\x4b\x24\x53\x2b\x9d\x48\x93\xc4\xb1\xb2\x75\x8a\xa5\x46\x75\x95\x5b\xa9\x5d\xdf\xa2\xdf\x4a\xb3\xb9\xad\x12\x74\x65\x8f\x00\x48\x99\xa9\x03\x80\xb5\x9e\x6d\x41\xf9\x17\x55\x79\x2b\x64\x97\x57\xe8\xd3\xbb\xb3\x5f\x91\xc9\xc1\xa2\xd3\x6c\x3a\x1e\x84\xdd\x4f\x46\xbf\x7e\x45\xdf\xfd\xa0\x7a\x38\x9a\xe9\xec\xb2\x49\xe0\x74\x42\xa3\x79\x60\x3a\x32\xc3\xdc\xae\x77\x3e\xe0\x0c\xe2\xa7\x55\x2e\x8c\xbb\x9c\xcf\x50\x5e\xde\xe6\x2c\x1f\x17\x04\x45\x42\x2a\x22\xa9\x30\x19\xc2\x1c\xc2\x18\x53\x88\x4c\x5e\x50\x92\xa1\xfb\x03\x31\x09\x68\x5c\x2d\xca\x0c\x03\x00\x52\xb2\x05\x25\x4c\x83\xe7\x33\xcc\x25\xe7\x31\x84\x29\x41\x59\xce\xea\x02\x2f\x49\x26\x7b\xc2\x68\x92\xdf\x5b\x38\x40\x05\x2f\xb5\x59\x89\xeb\x1a\x02\x2e\x2b\xe8\xda\x44\x79\x1b\xf8\x62\xe0\xba\x19\x54\xb1\x59\x37\xac\xfa\xb9\x3c\x14\x5a\xe6\xc4\x52\xcd\x89\x51\x91\x34\x5a\x94\x90\x2d\x13\xf4\x81\xa9\xd5\xd2\x0b\xab\x26\x5c\x5f\xbb\x1d\xa0\x23\xa9\xcd\xd4\x8c\xb4\x7a\x31\x2a\x47\x55\x08\x76\x60\xaf\xd7\xbf\xab\xee\x50\x4a\x09\xdc\x91\x99\x11\xb0\x6d\x7c\x21\x6e\xa5\x9e\x76\xad\x1f\x99\xe4\x43\x62\xa0\xc2\x10\x07\x0e\xf3\x9b\xf5\x4f\x26\x38\x1d\xd8\xa3\x23\x47\xb0\xc1\xbf\x21\xf3\x9d\xc6\xdd\x1e\xa8\xe3\x9e\xda\x7e\x66\x7c\xb6\xa1\xcd\x2f\xe2\x3d\x38\xc7\xfe\x7c\xd8\x43\xcf\x4c\x3b\xb9\x2b\x23\x74\x10\xc8\xe9\xf2\xa3\x8a\x0c\x8d\xd0\x00\x45\x45\x5e\x12\xed\xa9\x86\xdd\x5f\x5d\x15\x58\xf9\x72\xc4\x3b\x4c\x95\x7b\x5a\xfb\x6b\x0c\xbf\xab\x98\xf6\x5c\xd4\xc4\x0b\x5e\x45\x3d\x8f\xa8\xaf\xf2\x32\x83\xdb\x46\x8c\x28\xce\xec\x30\x34\xc7\xf7\xfd\x79\x5e\xee\xad\xc9\x36\x23\x94\x2e\xa7\xd6\xb4\xe8\xf7\xd1\x2f\x33\x52\xea\xb4\x32\xc2\x2e\x94\x17\xba\x33\xb3\x16\xcf\xf1\xbd\x5d\x8b\x37\xc8\x22\xb7\xde\x25\xc3\x2d\xa2\x7d\xba\xa0\x54\x96\xbf\x75\x21\xc9\xec\x51\x6a\x05\x0b\x43\x14\xa5\x1f\xc4\x8a\xdc\xf4\x81\x9a\x17\xc9\x12\x9d\x34\x3a\x78\xf2\x04\xb9\xaf\x1f\x85\x1c\x7c\x4d\x94\x9c\x06\x01\x2f\xad\x59\x4a\x05\x25\xf6\x47\x7e\x6b\xc5\xed\xee\x82\xe1\xf1\x72\x22\xc9\x37\xc7\xf7\x4f\x8f\x92\xc3\xef\xd7\x57\xcb\x4b\x4d\x1b\x6f\xa5\x87\x19\x80\x77\x67\xe5\x24\x2f\x73\xbe\x1c\x36\x66\xe6\xc0\x7f\xf1\xc0\x19\xfa\xfb\x4c\xc2\x31\xe0\xb8\x0b\xe9\xe5\x58\x36\x12\x3c\x34\xc7\xf3\x1d\x67\x76\xbe\xfb\x7c\xae\x9c\x9c\x0c\x80\xd5\x08\xa6\xa9\x19\xf4\x17\x9e\x4c\xb4\x6f\xfd\xcd\x6b\x67\x53\xfc\x1e\xe8\x7a\xa1\xb4\x56\xeb\x81\xc7\x87\xc9\xd1\xd3\xd8\x5c\xd1\x14\x85\x07\x02\x5e\xd7\x6e\x4a\xb6\x74\xbb\x15\xc2\x4a\x3b\xd5\x04\x2b\xdd\x2b\xd3\xa4\xad\x77\x13\x30\x7f\xe0\x84\xe0\x8b\xd4\x32\x83\x90\xca\x76\x6e\x73\x2f\xb7\xc0\xfa\x4d\xa9\xf2\xb5\xc0\xa4\xde\xab\x68\x4e\x4a\x6e\x34\x25\x99\xe8\xa8\x7b\x9e\xa7\x37\xaf\x54\x5e\x3c\xb8\xd2\x22\x93\xe4\xfd\xdb\xdb\x9f\x2e\x7a\x81\x35\x02\xd0\x51\x6b\x84\x7b\xe5\xdb\x27\x9d\xca\x79\x6e\x47\x31\xab\x6e\x09\x7d\x49\x38\xce\x8b\xf0\x58\x5e\xdb\x0a\xbb\x0d\x48\xa2\xe9\xdd\x3e\x89\xa5\xce\xef\xa1\xfb\x1e\x5a\xfa\x6a\x53\x85\x3c\x75\x8e\x59\x8d\x4b\x6d\x2a\x8a\xc2\xe8\xa4\x83\xf6\xed\x01\xce\x3d\x7a\x0a\x06\x5c\x37\xe1\xd5\xa7\x8b\x17\xd2\xb1\x13\x77\xd1\x3e\xea\x1c\xf7\x45\xdb\x93\xce\xd0\x01\xcb\xee\x30\x4f\x67\x6d\xc0\x30\x8e\x6b\xf9\x36\x92\x69\xb7\x46\xd1\x18\xa7\x37\x53\x2a\x4c\xa2\x03\xb5\x3b\xec\xc0\xee\x06\xd4\x05\x94\x88\x6e\x84\xe5\xda\xee\x28\xad\x4a\xae\x62\x24\x64\x97\xfb\x48\x8d\x36\x09\xf9\xd3\xc0\x30\x93\x4e\xb5\x01\x72\x1d\x8c\x4b\x35\x12\x59\x62\xba\x70\xc2\xbb\xa0\xc2\x98\x02\x59\x74\xaf\x4e\x91\xf2\x0a\x5b\x1f\xaa\x8f\x46\xdb\x5e\x01\x6f\x84\x4e\x6f\x1a\x98\xf8\x37\xf0\x2e\x68\x8f\xc8\x66\xc6\x20\xd9\xc8\x10\x4e\x6f\xb3\x7c\x3a\x2b\x84\x69\x02\x29\x52\x03\x5d\xfe\x44\x66\xf8\x36\xaf\x68\xa2\x54\xf5\x6b\xdd\x20\x46\x3b\xb1\x9e\xc4\x6b\xa0\xfe\xfa\x9d\xb3\x19\x29\x6e\xe5\x31\xc2\x0e\x3d\xcb\xdc\x6a\xbb\x31\xfc\xba\x5e\xdd\x48\x02\x93\x27\x65\xab\x13\x9c\xe5\x7f\xfb\x96\x2d\xa7\xaf\xa6\x9a\x07\x7e\x01\x4d\x60\x36\x05\x26\x14\xe1\x5b\x4d\xc4\x0d\x56\x81\x55\x37\x3b\x04\x74\x07\x82\x43\xb6\x04\x6b\x84\x69\x22\xf6\xd6\x0a\x0b\x95\x21\x90\xa1\x1a\x33\x06\x37\x86\x6d\x02\xc1\x49\x45\x8d\x3d\x28\x37\x3c\xe0\x30\x75\xb2\x06\x32\x7c\x4b\xf6\xd4\xae\xc8\xc9\x15\xf8\xfc\x5f\x9f\xff\x6a\x32\x68\x89\x5d\x4c\x45\x33\x42\x65\x9a\xc1\x03\xe3\x13\x45\x39\x97\x6e\x5b\xa7\x4f\x09\xec\x4e\x58\xa2\x02\xe2\x82\x11\x2a\x36\x58\x62\x7f\x24\x2f\x9c\x01\x3e\x6e\x5e\x68\x93\x62\x50\xf9\x1b\xbd\x8d\x62\x38\x35\x21\x38\x5f\xb7\xba\x23\x82\x5e\xd3\x77\x15\xa0\x09\xee\x21\x86\x26\x42\x23\x36\x3c\xa1\x6d\xbf\xc0\x05\x1e\xfb\x99\x25\xdd\x64\x41\xce\x09\x91\x49\x61\xb8\x13\x17\x34\x42\x6f\x1a\x01\xab\x78\x27\x3e\x90\x81\x88\x36\xeb\xd1\x66\x2c\x5d\x4a\x4b\x7f\xb8\x3e\x20\xf9\xa9\xca\x96\x9a\xd4\x0e\x38\x3f\x53\xfb\x35\x24\x07\x40\x7c\x5c\x65\x2a\x47\x27\xb4\xf3\xa2\x14\xd9\x5d\xce\xd3\x59\xdc\x38\xff\x57\xf7\x94\x31\x23\x28\xba\x25\x29\xaf\x68\x34\xd8\x73\xcd\x43\x73\x09\x56\x09\xb5\x58\x48\xda\xc7\xf2\xfe\xcc\xea\xee\x95\xb3\x24\x3a\xe6\xf4\xe4\x98\x67\x28\xad\x0a\xb1\x86\x8d\x3a\xcf\x3a\x27\xc7\xf9\x49\x29\x27\xfc\xb8\x9f\x9f\x1c\xf7\x79\x26\x7e\xe8\x49\xd4\xb8\x79\xe9\x6e\xa5\x3d\x7c\xd4\x62\x90\xcf\x73\x2e\xd5\xa6\x8b\x6e\xb7\xe1\xd5\x74\xae\x26\x05\x62\x0a\xfc\x0b\xf6\x30\x99\xca\xc0\xd5\x29\x3e\xf2\x2b\x77\xd9\x35\xa7\x56\x21\xd7\xb6\xf1\x6c\x0f\x37\xd1\xe2\xa4\x71\x7e\x27\x41\xaa\x53\x36\x41\x0b\x55\x45\x79\xae\x2f\x8f\xae\xec\x2b\x97\x4c\x92\x30\x70\x6b\x74\x68\x26\x52\x1d\x4f\x04\x27\xf2\xff\xd1\x09\xbb\xfd\xf6\x09\xbb\x6d\x4e\x98\xb9\xd1\x77\x41\xee\x05\xde\x91\x39\xfc\x30\xe8\x7d\x96\xe8\x7d\x46\xc7\xe8\x56\x9f\x2d\x68\xdc\x3e\xfb\xd9\x1a\x2c\xa4\xfd\x91\xa9\x7c\xf9\xf9\x4a\x4d\x29\xfa\x1f\x62\x9a\xdd\xf2\x43\x39\xd5\x63\xda\x3f\x89\x7c\x07\xf3\x1f\xe4\x25\x07\x93\x9d\x59\x49\x9d\xfe\x48\x56\x0a\xf7\x2e\xab\x78\x3d\xb9\x33\xb1\x8e\x73\x9b\x1d\x81\x4d\xbd\xb9\x23\xa8\xe2\x75\xe4\x8c\xda\xef\xb3\xbb\xa5\x53\xe5\x20\x1d\x04\x57\xa2\x4f\x25\x5b\xd4\x75\x45\x39\xc9\xd4\xd5\x4c\x38\xb9\x6b\x01\x59\x7d\xb3\x9d\x15\xfe\x7a\x5c\x28\x89\x4b\xf3\xb3\x4f\x9e\x93\xdc\xe9\xfc\x63\xb8\x78\x67\x9c\xec\xfe\xce\xc5\x6b\x69\x11\xc3\x63\x76\xbd\x74\x53\x1a\x2d\xcd\x3a\x2f\x5f\x9d\x8c\xd0\x11\x79\xf6\x2f\x8d\xdb\x2e\xf1\x12\xf5\x65\x79\xc2\x2b\x67\xe3\x14\xfd\x16\x0d\xfd\xe4\xe1\x2e\x94\xa3\x35\x50\x8e\x9a\x50\xfe\x63\x03\x94\xa3\x3f\x87\xa1\x1c\xfd\xb9\x09\xe5\x74\x13\x94\xef\xd7\x40\xf9\xbe\x09\xe5\xc3\x26\x28\xcf\xd6\x40\x79\xd6\x84\x72\xb1\x01\xca\x5f\xc2\x40\xfe\xd2\x84\xf1\xd7\x0d\x30\x7e\x08\xc3\xf8\xa1\x09\xe3\xed\x06\x18\xcd\xdb\xd0\x0a\xc6\x77\x4d\x18\x37\xeb\x61\x34\x20\x2c\x43\xf5\xbc\x35\x6a\x53\xc5\x63\x81\xd4\xc1\x3a\xde\x3b\x68\x33\xdf\x32\x8c\x98\x82\xb3\x86\xfb\x0e\xda\xec\xf7\xb7\x4d\x70\xd6\xf1\xdf\x41\x9b\x01\xf1\x46\x38\x6b\x38\xf0\xa0\xcd\x82\x93\x8d\x70\xd6\xf0\xe0\x41\x9b\x09\xeb\x4d\x70\xfe\x62\x97\xb7\x06\xa0\x16\x23\x96\x9b\xe0\xac\xe1\xc4\x83\x16\x2b\xfe\xaf\xff\xb9\x0e\xcc\x11\x39\x58\xc3\x8b\x07\x2d\x66\x9c\xaf\xc7\x25\xc4\x63\x5b\x32\x65\x38\x76\x8c\x97\xc7\x40\x5a\x33\x9b\xc2\x4f\xde\x3e\xff\xf5\xfa\xfc\xf4\xe3\xd9\xe9\xf9\xf5\xbb\x4f\x6f\xd5\x87\x51\xed\xad\x17\xc2\x18\x86\x8b\xe0\xd1\xfa\x1c\x6d\xaf\x08\x4f\x67\x4e\x8a\x36\xdf\xb4\xdb\x87\x04\x6c\xd2\xe7\x35\x5e\x70\x7d\x8a\x96\x97\x53\x54\x95\xc5\x12\x4d\x72\xca\xb8\x69\xdb\x40\x67\x1f\x45\x49\x64\xa2\x09\x7d\xc0\x27\x8d\xca\xad\x9d\x9c\x8e\x92\x57\x83\xf0\xc3\x58\x14\x2c\x56\x17\x79\x4a\xe2\xc3\x5e\x13\x58\x23\x06\x48\x56\x87\x84\x25\x32\x8d\x92\x4d\x73\xa4\xae\xe6\xd8\x64\x47\x03\x74\x09\xde\x20\xb9\x93\xd6\x4f\xcd\x44\x4a\x03\x99\x03\x12\xce\x53\xe3\x70\xd6\x67\x95\x2e\xb2\x23\x4f\x54\x05\x98\xc6\xb2\xf9\x01\xb7\x2e\xe9\x07\x6a\x34\xb2\x44\x85\x57\x7a\x40\xd6\x66\x5b\x01\x2f\x2b\xe0\xf6\xe9\xe3\x1b\x7b\xc6\xee\xd6\x0a\xda\xee\x5e\x05\x79\x64\xb8\xb2\xc1\x9c\xde\x5b\x7d\xee\x00\x5d\xe1\x2c\x93\x6e\x24\x64\x3e\x83\xf7\x38\x8e\xfe\x84\xb3\xec\x5a\x7d\xb6\x48\xa5\xfd\xf4\x6a\xcb\xcf\x43\x89\xa2\x1e\xfa\xb2\xea\xb6\xed\x8a\xc6\xf0\xf5\x80\xda\x24\x10\x83\x53\xe1\x9f\x62\x2e\x20\xbf\x3d\x23\x98\xca\x2f\x13\x46\x51\x43\x26\x75\x10\x94\x22\x1e\xcc\xed\x07\x7d\xe3\x25\x0c\x27\x61\x8b\xb1\xb4\x19\xe3\xa3\x2e\xf0\x1d\x8f\x3b\x4f\x3a\xe6\x2a\x9b\x85\xf1\x9a\x14\xb5\x71\x0b\x36\x07\xf3\xef\x8d\x6a\xb1\x1b\xca\xd1\x84\x21\x07\x6c\x9b\xb0\xd8\xc1\x74\x2b\xb5\x34\x95\x5d\x6a\xe9\x4f\x82\xfa\x7c\xd3\xc6\x55\xba\x38\x80\x64\x8f\xcd\xe7\x38\x9d\x0f\xb2\x29\x87\xbf\xfa\x58\xa9\x34\xb3\xc5\xcc\x4a\x07\xc9\xa7\x8f\x6f\xec\xd4\x76\x9d\xd7\xd2\xdc\x6c\xcc\x7d\x77\x0f\x12\x27\xee\x79\xc9\xf3\xa4\xf0\x49\x16\xd4\xf1\x13\xc0\x54\x72\x76\xba\xca\xbd\xd8\x8e\xf0\xd5\x51\x21\xc6\xf9\x68\xbf\x35\x21\xc8\xd5\xef\xa3\x77\xef\x2f\x4e\x07\x8d\x5c\x4b\x63\x82\x6e\x48\x0d\xa9\xd7\xd9\xb2\x4c\x65\x84\x40\x7f\xc1\xf3\x42\xa8\x49\xfd\x37\xad\xca\xdb\x64\x5a\x0d\x00\xee\x9b\xbc\xbc\x79\x55\xd1\x53\x13\x69\xb7\x61\x2a\x0c\x59\xc2\xc2\x0b\xb3\x2a\x57\x17\x88\xba\x6c\x52\xc1\x0b\x34\x9b\x4a\x39\x83\xc4\x43\x6e\x70\x5e\x43\x03\x48\x3a\xd8\x14\x4c\x3a\x42\xe6\x0f\xf3\xaa\x03\xe2\xfd\xf8\x33\x49\xb9\xce\x45\xe7\x32\xee\x94\x94\x84\x62\x2e\x79\x57\x56\xf3\x94\x8f\xc6\xdf\xd3\xe9\x8f\x65\x04\x56\xec\xc0\xd6\x41\xd8\xf2\xf3\x9f\x32\xf6\xf5\x89\xfa\x82\x98\x52\xaf\xc0\x23\xe7\x1c\x73\x12\x7f\x59\xf5\x50\x14\xf5\x90\x8c\xe7\xf9\x51\xec\xe9\x1c\xd2\x6e\x15\x18\x87\x3b\xdd\x79\x92\xec\x07\x94\x6e\x4d\x4d\x70\xba\x54\x6a\x37\x0b\xa0\x8b\xbe\xa8\x21\x4e\xc1\x9f\x0d\xf5\x02\xf7\x34\x82\x54\xdf\x61\x81\x68\x36\x69\xaa\xcc\x7f\xf7\xf4\x9b\x81\xe6\x2a\x13\xc3\x8b\xe0\x01\x26\x99\xdf\x44\x1e\xda\xc1\xb0\xce\xca\x5b\x5c\xe4\x59\x40\x1f\xc9\x14\x6c\xae\x3e\x93\xcd\x84\x79\xa1\xa6\xfd\x15\xad\xe6\xef\x65\x07\x0a\x40\xbb\xbb\x1e\x3a\xdc\x91\x32\x89\xed\x5d\x9e\x2e\xa2\x11\xea\xff\xe7\xf4\xf7\x6c\xff\xf7\x24\xd9\x1f\x25\xfb\x8f\xfb\x0f\x23\x56\x60\x84\x2e\xbd\x80\x3b\x2f\x16\x75\xa1\x8f\xe3\xd5\x30\x9d\xf2\xd6\xdc\xdb\x77\x8d\x25\xe8\xc1\x83\x4b\x38\x61\xdc\x85\x37\x0c\x5f\xf6\xd9\x3a\xc8\x4d\xf3\xb1\x86\x3d\x7a\x92\x65\xcf\xac\xce\x11\x0b\xae\x53\xc1\x1a\x13\xad\x3d\x56\x63\xad\xad\xe1\x6b\xd9\xef\x27\x42\xff\x02\x3c\x2f\xc5\x22\x40\x93\x1f\xd4\x8e\x9d\x2e\xf5\x22\x5b\x2e\xe6\x63\x42\xdf\x4f\x64\xa7\xaf\x2a\x2a\xa0\x68\x81\x75\xd1\xd9\x79\x1a\xec\x0b\x19\x9c\xca\x7e\xc9\xf9\x2c\x6e\x21\xa9\x88\x6d\xee\x8d\x29\x0a\x6c\xc2\x67\x3b\x25\xb6\x0d\xc2\x1a\xb7\xeb\xfb\xe9\x3a\x97\xc6\x1b\xa0\xda\x85\xfe\x42\xb2\x13\x4d\x8c\xd1\xd3\x22\x89\xa2\x85\x61\xc3\x56\x06\x3f\x6b\x83\x3a\xd2\xfd\x7e\xf2\xbe\x54\xeb\x72\x1b\x3f\x33\xcf\x12\xc8\xf3\x34\x5d\xcc\x17\x05\xe6\x70\x59\x6c\x07\x65\xb2\x86\x63\xd1\xbe\x4a\x39\xd0\x02\x6b\x62\xef\xec\x87\xd6\x9b\x69\xe8\x9c\xda\x0f\x16\xb5\xf5\x83\xdf\xae\x86\xbd\x44\x88\xc8\x67\xee\x56\x98\x90\x3b\x89\xb6\xf5\x3b\x3c\x27\xcf\xcb\x4c\xdf\xf5\xe0\x72\x46\xa5\xe5\x3a\xea\x38\x8b\xb9\xad\x8e\x46\x81\xb6\x90\x93\xbc\x51\x59\x03\xcd\x48\x5a\x65\xe4\xd3\xc7\xb3\x17\xd5\xbc\xae\x4a\x52\x6a\x5a\x7a\x00\x8e\xae\x6c\x76\xd3\xdf\xf7\x21\xbd\x29\x8a\xba\x3a\x5d\xb9\x90\x24\x17\x85\x11\x8a\x38\x1e\x3b\x57\x6a\xfc\x2e\x4d\xa2\x0a\xa7\x58\x7e\xad\x80\xe3\x31\xca\x19\xc4\xec\x4d\x09\x55\x7e\x68\xd7\x52\xbd\xb4\xdd\x5c\x99\xa1\xfe\xac\xf3\x23\xae\x02\xd3\xdf\xce\x42\xb8\x6d\xd2\x9b\x7a\xcc\x9d\x6a\xc7\x68\x53\xbd\x44\x53\x61\xa5\xe4\x8a\x4d\x61\xb7\xfc\xc0\xfe\x02\xa6\x56\xcb\x7a\x69\x58\x5d\x86\xcb\x6a\x8d\x61\x58\x03\xe7\x9e\xf2\xf5\x4d\x3e\xc9\x96\xf2\x31\xb9\x21\x4b\xe6\xf5\xd4\x6d\x33\xe9\x8d\xfd\x1a\xbc\x03\xe9\x52\xa1\xb0\x8f\x6e\xc8\xf2\x4a\xdb\xad\x0a\xca\xa5\x28\x6b\x05\xbc\x3b\xad\x8d\x4d\x7f\x31\x23\x8c\x20\x7e\x57\xa9\x2c\x04\x0c\xc5\x39\x7b\x49\x6a\x4a\x52\xcc\x89\xdc\x07\x09\xf3\x1b\x97\x19\xa2\x24\xcb\x29\x49\xf9\x45\xf5\x36\x9f\x0a\xca\x65\x9f\x3e\xbe\xe9\x0a\x28\x98\x12\x84\xb3\x8c\x64\xca\xb5\x51\x51\xf8\xf0\xf1\x1d\xa6\x19\xdc\xf8\xc6\x3c\x1f\xe7\x45\xce\x97\x62\xcb\x50\x15\x3a\x27\xbc\xf4\x76\x27\x7b\x26\x43\x6d\xa8\xeb\x0d\x1b\xd5\x19\x66\xb3\x0d\x0b\xa8\xfd\xfc\x85\xd6\xb1\x52\xe8\xb2\x57\x14\x4f\x55\x56\x93\x80\x18\x86\x7a\x91\xa7\xb9\x74\x69\x24\xcb\xf1\x61\x34\x80\x2a\xd5\x1f\x1f\x75\xa5\x6c\x65\xb4\xaa\xe1\x60\x5f\xc0\x41\x7f\x02\xe7\x57\x0a\x61\x42\x31\x69\xb9\xf0\x1c\x94\xad\x31\x48\x85\x94\xad\xf6\x2c\x99\x82\x13\xe1\x6c\x8b\xfe\xd8\x30\x03\x7b\xa2\x3f\x32\xda\xb0\x04\x34\x9d\x22\xde\x02\x5b\xf9\x52\x67\xd5\xb3\x11\xbb\x80\xf4\x8b\x3a\xae\x54\x55\xbb\x08\xd4\x66\x91\xaa\x1a\xd2\xa4\xe5\x69\xe5\x8b\x36\x64\xf4\x08\xef\xc0\x1a\x44\x0e\x64\x29\x69\xec\xb8\x60\xa2\xfb\x4f\x9f\xee\xa1\xa7\x48\x26\x61\x52\x59\x2b\xd0\x4c\x6e\x6a\xcc\xa7\x1e\x45\x8d\xa7\x4f\xfb\xca\x2d\xe7\xa6\xbb\x50\x8e\x39\x93\xda\x3c\x90\x9b\x5d\xb1\xd6\x56\x4f\x9c\xfa\xa0\x15\x40\x3f\xb0\x6e\xb9\x3d\x9d\x65\xba\x95\xe0\x78\x10\xdc\xfe\xe9\x1c\xec\xfa\x0b\x8e\xf2\x66\x48\x96\xdf\x26\x0d\xc8\x43\xa7\xb2\xfa\xfe\x9c\xfb\xdd\x47\x73\xb7\xa5\x93\xeb\xba\xfa\x65\x52\x4d\x26\x71\x07\x7c\x65\x1d\x77\x7d\x5c\x97\xed\xdd\x39\x74\x17\x2a\xdc\x7c\x66\xcc\x74\x36\x74\x97\x41\xdb\x4b\xa9\x3b\xe9\x85\x32\xde\xc3\x1d\xcf\x07\x7d\xe4\xac\x91\xa9\x64\x1d\x1e\x9e\xcb\x7c\xf7\xef\x96\x85\xa1\x4f\x26\x01\xf0\x2e\x7f\xc3\xdc\x7a\x49\x53\x36\xcf\xab\x16\x03\xe9\x3b\xf1\xb3\x17\x9b\xe9\x64\xe7\xf9\xbc\xd6\x1f\xcf\xf0\xf7\xff\xce\xd7\x16\x40\xa6\xdf\x4f\x62\x95\x67\x5e\xec\xfa\x0f\x8e\x6c\xf8\xb2\x0f\xa5\xa9\x3f\xbd\x39\x53\x1f\x35\x64\x6a\x93\x6c\x85\xe3\x01\xfe\x67\x77\x00\x33\xcc\x44\x4d\x05\x4e\xdb\x53\x1e\xf0\x6d\xd8\xfb\x20\x2c\xf6\x01\x48\x6a\x0f\xb3\xb5\x0f\x9d\x15\xcc\x0c\x3b\xd0\x02\x9c\x81\xb2\xfa\xb0\x39\x6b\x2f\xaa\x05\xac\x10\x81\x66\x6e\xae\x8f\x20\x21\x43\x68\x03\xd6\x0e\xe4\x03\xf4\xfd\x61\xcf\xe9\x4a\xcb\x66\xf0\x43\x51\x86\xf2\xad\x0f\x45\xb5\xbb\xd2\x93\xb3\xe1\x9b\x0d\x41\x14\x87\x0f\x55\x0d\xd2\x07\x69\xbe\x31\x21\x61\x9f\x13\xef\x53\x10\x0f\xfb\x5a\x84\x1b\x88\x0b\xa2\xa6\xa5\x7e\xd0\xfe\x08\xae\x46\xe6\x1f\x87\x8a\x72\x56\xaf\xd1\xb2\x3b\x7c\xf7\x77\xd3\x67\x83\x1b\x4a\xda\xd6\xcc\xd9\xc1\x2e\x1f\x0e\x0e\xaf\x40\x3d\x79\xb3\x67\xd8\x24\xe0\x64\xf2\xf7\xa0\xa0\x43\x9e\x6f\xa5\x8e\xfe\x68\xf2\x96\x0f\x15\x37\xa8\xe3\x01\xfb\x83\x04\x02\x4b\xd2\xa1\xd0\xba\x81\x3b\x04\x9b\xeb\x32\x9b\xa0\xae\xe5\xa1\x6d\xda\x52\xde\x62\xe0\x9c\x95\xc9\x45\x40\xe6\x0b\x35\x9f\x50\xf0\x57\xa6\x4d\xf5\x12\xf3\xb5\x16\x83\x54\x70\xa1\xd2\xdb\x98\x97\x64\x42\xa8\xd8\xc0\xde\x6a\x4b\xab\x9a\xa0\x49\x09\x3b\x91\x3b\x9c\xf3\x0f\x84\xe6\x55\x26\xd0\x93\x0b\x05\xb1\x9f\x73\x10\x1b\xdf\x14\x17\x05\x11\x1b\x9c\x9a\x08\x6b\xba\x58\x5a\x5b\x3b\x23\xe3\x6a\x51\xa6\x24\x9e\x94\x3d\x07\x94\x93\x3c\x43\xa6\x4c\x71\xf6\x59\x2d\xbb\xc0\x4b\xf8\xa2\xea\x3b\x5e\x04\x4c\x21\x91\x06\xa6\xd3\x85\x30\xa9\x9d\x0f\xcc\xb1\xb4\xaa\x89\xf7\xd9\x39\x8b\x75\x79\x5b\xdd\x90\x57\xae\xf1\xa1\xbb\x2f\x13\x5c\xd7\xc5\x32\x86\xd6\x3d\x00\xef\x05\x97\x29\x0c\x20\x10\xcf\x64\x7c\xd1\xf0\xbc\x41\x0e\x65\x76\x45\x77\xef\xe1\xa6\x14\x90\xa7\x03\x86\x12\xa4\xe4\x39\x5f\xbe\x95\x5f\x74\x81\x9e\xa2\x27\xd1\x00\x45\x4f\xf0\xbc\x1e\xea\x0f\x47\x1c\x43\x49\xc1\x4d\xc1\x09\x14\x4c\x4d\x41\x27\xea\x0c\x50\xe7\xc9\x7f\x2d\x2a\x3e\xec\xa8\x3a\x9d\x48\x14\xfd\xe9\xbb\xbf\x98\x92\xbe\x2c\xb9\x7f\xf6\x6a\xd8\x31\x29\x20\x15\x01\x54\xb8\x81\x42\xcf\x3a\x50\x2e\x9f\x1c\x9f\x44\x9d\xdf\xfb\x57\xfd\x69\xcf\xf9\x2a\x0a\x6b\x18\xc5\x66\x18\x97\xcc\xec\xa1\x5d\x0a\xe4\x65\xce\x15\xd5\x65\x22\xbf\x73\xc2\x17\xb5\x0a\xa1\x4f\x71\x3a\x23\xea\x6b\x3e\xd6\x33\xe2\x25\xfc\x0b\x7e\xed\x8b\x71\xcc\xf3\xb4\xff\x99\xc9\x0d\xc1\x35\x27\xf3\xba\xc0\x5c\xc7\x42\x8f\x31\xfd\xf1\x76\x24\x76\x09\x3f\x7d\x3a\x7b\xf3\xf2\xfa\xe7\xd3\x8f\xe7\x67\xef\xdf\xa9\xdb\x94\xad\xb4\x7e\x42\x9e\x04\x86\x7b\x8e\xa4\x5d\x28\x88\x2a\x44\x53\x8b\xd3\xdb\x05\xe3\x02\x67\xbd\xbd\x13\x2d\x87\xae\x35\x1b\xdc\x92\xfb\xf9\x1a\x82\xbb\xd1\x86\xdd\xea\x86\x62\x52\xa4\x0e\xf0\x03\x87\x67\xb1\x93\x15\x00\x3c\x38\x92\xdc\x0d\xe9\xb7\xd3\xf2\x38\x16\x15\xba\xc3\xbd\xff\x1d\x00\x00\xff\xff\x38\xdd\x68\xd2\x42\x97\x00\x00") func pkgUiStaticJsGraphJsBytes() ([]byte, error) { return bindataRead( @@ -520,7 +520,7 @@ func pkgUiStaticJsGraphJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 37987, mode: os.FileMode(420), modTime: time.Unix(1572558524, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 38722, mode: os.FileMode(420), modTime: time.Unix(1576074054, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/pkg/ui/static/js/graph.js b/pkg/ui/static/js/graph.js index b191477dcd..18f288de2a 100644 --- a/pkg/ui/static/js/graph.js +++ b/pkg/ui/static/js/graph.js @@ -194,46 +194,62 @@ Prometheus.Graph.prototype.initialize = function() { self.updateGraph(); }); + // Toggle functions. + self.toggleOn = function(targetEl, item) { + $(targetEl).addClass('glyphicon-check').removeClass('glyphicon-unchecked'); + localStorage.setItem(item, '1'); + }; + self.toggleOff = function(targetEl, item) { + $(targetEl).addClass('glyphicon-unchecked').removeClass('glyphicon-check'); + localStorage.setItem(item, '0'); + }; + // Deduplication. + let dedup_icon = self.dedupBtn.find('.glyphicon'); self.isDedupEnabled = function() { - return self.enableDedup.val() === '1'; - }; - var styleDedupBtn = function() { - var icon = self.dedupBtn.find('.glyphicon'); - if (self.isDedupEnabled()) { - icon.addClass("glyphicon-check"); - icon.removeClass("glyphicon-unchecked"); - } else { - icon.addClass("glyphicon-unchecked"); - icon.removeClass("glyphicon-check"); - } + let v = localStorage.getItem('enable-dedup'); + + // If not set in localstorage, make it enabled. + return v === '1' || v === null; }; - styleDedupBtn(); + + if (self.isDedupEnabled()) { + self.toggleOn(dedup_icon, 'enable-dedup'); + } else { + self.toggleOff(dedup_icon, 'enable-dedup'); + } self.dedupBtn.click(function() { self.enableDedup.val(self.isDedupEnabled() ? '0' : '1'); - styleDedupBtn(); + if (dedup_icon.hasClass('glyphicon-unchecked')) { + self.toggleOn(dedup_icon, 'enable-dedup'); + } else if (dedup_icon.hasClass('glyphicon-check')) { + self.toggleOff(dedup_icon, 'enable-dedup'); + } }); // Partial response. + let partial_response_icon = self.partialResponseBtn.find('.glyphicon'); self.isPartialResponseEnabled = function() { - return self.partialResponse.val() === '1'; - }; - var stylePartialResponseBtn = function() { - var icon = self.partialResponseBtn.find('.glyphicon'); - if (self.isPartialResponseEnabled()) { - icon.addClass("glyphicon-check"); - icon.removeClass("glyphicon-unchecked"); - } else { - icon.addClass("glyphicon-unchecked"); - icon.removeClass("glyphicon-check"); - } + let v = localStorage.getItem('enable-partial-response'); + + // If not set in localstorage, make it enabled. + return v === '1' || v === null; }; - stylePartialResponseBtn(); + + if (self.isPartialResponseEnabled()) { + self.toggleOn(partial_response_icon, 'enable-partial-response'); + } else { + self.toggleOff(partial_response_icon, 'enable-partial-response'); + } self.partialResponseBtn.click(function() { self.partialResponse.val(self.isPartialResponseEnabled() ? '0' : '1'); - stylePartialResponseBtn(); + if (partial_response_icon.hasClass('glyphicon-unchecked')) { + self.toggleOn(partial_response_icon, 'enable-partial-response'); + } else if (partial_response_icon.hasClass('glyphicon-check')) { + self.toggleOff(partial_response_icon, 'enable-partial-response'); + } }); self.maxSourceResolutionInput.val(self.options.max_source_resolution); diff --git a/pkg/ui/templates/query_menu.html b/pkg/ui/templates/query_menu.html index f65676fc98..0497d3b3d4 100644 --- a/pkg/ui/templates/query_menu.html +++ b/pkg/ui/templates/query_menu.html @@ -1,25 +1,25 @@ {{define "nav"}} {{end}} From 698c7a922cd5dcaa53e305db5c2adb4e79d328ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Mon, 13 Jan 2020 16:11:37 +0200 Subject: [PATCH 171/257] *: update for 0.10.0 (#1989) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * *: update for 0.10.0 Signed-off-by: Giedrius Statkevičius * CHANGELOG: update date Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 2 +- VERSION | 2 +- tutorials/katacoda/thanos/1-globalview/courseBase.sh | 2 +- tutorials/katacoda/thanos/1-globalview/step2.md | 8 ++++---- tutorials/katacoda/thanos/1-globalview/step3.md | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 422cefdb47..0c74dbcc78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.) -## [v0.10.0-rc.1](https://github.com/thanos-io/thanos/releases/tag/v0.10.0-rc.1) - 2020.01.10 +## [v0.10.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0) - 2020.01.13 ### Fixed diff --git a/VERSION b/VERSION index 2fe3f360c0..78bc1abd14 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.10.0-rc.1 +0.10.0 diff --git a/tutorials/katacoda/thanos/1-globalview/courseBase.sh b/tutorials/katacoda/thanos/1-globalview/courseBase.sh index cad8757226..6bda9cb811 100644 --- a/tutorials/katacoda/thanos/1-globalview/courseBase.sh +++ b/tutorials/katacoda/thanos/1-globalview/courseBase.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash docker pull quay.io/prometheus/prometheus:v2.14.0 -docker pull quay.io/thanos/thanos:v0.10.0-rc.1 +docker pull quay.io/thanos/thanos:v0.10.0 diff --git a/tutorials/katacoda/thanos/1-globalview/step2.md b/tutorials/katacoda/thanos/1-globalview/step2.md index 569a9de690..4982909dfe 100644 --- a/tutorials/katacoda/thanos/1-globalview/step2.md +++ b/tutorials/katacoda/thanos/1-globalview/step2.md @@ -10,7 +10,7 @@ component and can be invoked in a single command. Let's take a look at all the Thanos commands: ``` -docker run --rm quay.io/thanos/thanos:v0.10.0-rc.1 --help +docker run --rm quay.io/thanos/thanos:v0.10.0 --help ```{{execute}} You should see multiple commands that solves different purposes. @@ -53,7 +53,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_eu1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-eu1 \ -u root \ - quay.io/thanos/thanos:v0.10.0-rc.1 \ + quay.io/thanos/thanos:v0.10.0 \ sidecar \ --http-address 0.0.0.0:19090 \ --grpc-address 0.0.0.0:19190 \ @@ -68,7 +68,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.10.0-rc.1 \ + quay.io/thanos/thanos:v0.10.0 \ sidecar \ --http-address 0.0.0.0:19091 \ --grpc-address 0.0.0.0:19191 \ @@ -81,7 +81,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus1_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-1-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.10.0-rc.1 \ + quay.io/thanos/thanos:v0.10.0 \ sidecar \ --http-address 0.0.0.0:19092 \ --grpc-address 0.0.0.0:19192 \ diff --git a/tutorials/katacoda/thanos/1-globalview/step3.md b/tutorials/katacoda/thanos/1-globalview/step3.md index 987c918091..cbe7d5f192 100644 --- a/tutorials/katacoda/thanos/1-globalview/step3.md +++ b/tutorials/katacoda/thanos/1-globalview/step3.md @@ -28,7 +28,7 @@ Click below snippet to start the Querier. ``` docker run -d --net=host --rm \ --name querier \ - quay.io/thanos/thanos:v0.10.0-rc.1 \ + quay.io/thanos/thanos:v0.10.0 \ query \ --http-address 0.0.0.0:29090 \ --query.replica-label replica \ From f08038b20914186a6abc1cd5ea5d2da8306e7b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Mon, 13 Jan 2020 16:51:46 +0200 Subject: [PATCH 172/257] *: update for 0.10.0 release (#1990) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * *: update for 0.10.0 release Clean up the `CHANGELOG.md` file, update the katacoda scripts, etc. for the `0.10.0` release. Signed-off-by: Giedrius Statkevičius * CHANGELOG: update date `0.10.0` was released on 2020-01-13. Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 6 +++++- VERSION | 2 +- tutorials/katacoda/thanos/1-globalview/courseBase.sh | 2 +- tutorials/katacoda/thanos/1-globalview/step2.md | 8 ++++---- tutorials/katacoda/thanos/1-globalview/step3.md | 2 +- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 657e7f2826..006087f904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,14 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased +### Added + - [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags - [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction -- [#1975](https://github.com/thanos-io/thanos/pull/1975) Store Gateway: fixed panic caused by memcached servers selector when there's 1 memcached node - [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`. +## [v0.10.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0) - 2020.01.13 + ### Fixed - [1985](https://github.com/thanos-io/thanos/pull/1985) store gateway: Fixed case where series entry is larger than 64KB in index. @@ -35,6 +38,7 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1872](https://github.com/thanos-io/thanos/pull/1872) Ruler: `/api/v1/rules` now shows a properly formatted value - [#1945](https://github.com/thanos-io/thanos/pull/1945) `master` container images are now built with Go 1.13 - [#1956](https://github.com/thanos-io/thanos/pull/1956) Ruler: now properly ignores duplicated query addresses +- [#1975](https://github.com/thanos-io/thanos/pull/1975) Store Gateway: fixed panic caused by memcached servers selector when there's 1 memcached node ### Added diff --git a/VERSION b/VERSION index c70836ca5c..29b2d3ea50 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.0-dev +0.10.0-dev diff --git a/tutorials/katacoda/thanos/1-globalview/courseBase.sh b/tutorials/katacoda/thanos/1-globalview/courseBase.sh index 196c8b5b9f..6bda9cb811 100644 --- a/tutorials/katacoda/thanos/1-globalview/courseBase.sh +++ b/tutorials/katacoda/thanos/1-globalview/courseBase.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash docker pull quay.io/prometheus/prometheus:v2.14.0 -docker pull quay.io/thanos/thanos:v0.9.0-rc.0 +docker pull quay.io/thanos/thanos:v0.10.0 diff --git a/tutorials/katacoda/thanos/1-globalview/step2.md b/tutorials/katacoda/thanos/1-globalview/step2.md index a5ee9470b7..4982909dfe 100644 --- a/tutorials/katacoda/thanos/1-globalview/step2.md +++ b/tutorials/katacoda/thanos/1-globalview/step2.md @@ -10,7 +10,7 @@ component and can be invoked in a single command. Let's take a look at all the Thanos commands: ``` -docker run --rm quay.io/thanos/thanos:v0.9.0-rc.0 --help +docker run --rm quay.io/thanos/thanos:v0.10.0 --help ```{{execute}} You should see multiple commands that solves different purposes. @@ -53,7 +53,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_eu1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-eu1 \ -u root \ - quay.io/thanos/thanos:v0.9.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0 \ sidecar \ --http-address 0.0.0.0:19090 \ --grpc-address 0.0.0.0:19190 \ @@ -68,7 +68,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.9.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0 \ sidecar \ --http-address 0.0.0.0:19091 \ --grpc-address 0.0.0.0:19191 \ @@ -81,7 +81,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus1_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-1-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.9.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0 \ sidecar \ --http-address 0.0.0.0:19092 \ --grpc-address 0.0.0.0:19192 \ diff --git a/tutorials/katacoda/thanos/1-globalview/step3.md b/tutorials/katacoda/thanos/1-globalview/step3.md index 80d8066ec3..cbe7d5f192 100644 --- a/tutorials/katacoda/thanos/1-globalview/step3.md +++ b/tutorials/katacoda/thanos/1-globalview/step3.md @@ -28,7 +28,7 @@ Click below snippet to start the Querier. ``` docker run -d --net=host --rm \ --name querier \ - quay.io/thanos/thanos:v0.9.0-rc.0 \ + quay.io/thanos/thanos:v0.10.0 \ query \ --http-address 0.0.0.0:29090 \ --query.replica-label replica \ From 17fd999eee5d26b8ed384ff1ccadb78533b01386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Mon, 13 Jan 2020 17:00:21 +0200 Subject: [PATCH 173/257] api/v1: add cache-control header (#1984) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * api/v1: add cache-control header Let's set `Cache-Control` to `no-store` if there were any warnings while the query was being executed i.e. it is a partial response or we got some other problem. Signed-off-by: Giedrius Statkevičius * api/v1: add missing check Signed-off-by: Giedrius Statkevičius * api/v1: don't store on errs Signed-off-by: Giedrius Statkevičius --- pkg/query/api/v1.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/query/api/v1.go b/pkg/query/api/v1.go index 17ebc07321..0fb1b24b44 100644 --- a/pkg/query/api/v1.go +++ b/pkg/query/api/v1.go @@ -536,6 +536,9 @@ func (api *API) series(r *http.Request) (interface{}, []error, *ApiError) { func Respond(w http.ResponseWriter, data interface{}, warnings []error) { w.Header().Set("Content-Type", "application/json") + if len(warnings) > 0 { + w.Header().Set("Cache-Control", "no-store") + } w.WriteHeader(http.StatusOK) resp := &response{ @@ -550,6 +553,7 @@ func Respond(w http.ResponseWriter, data interface{}, warnings []error) { func RespondError(w http.ResponseWriter, apiErr *ApiError, data interface{}) { w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") var code int switch apiErr.Typ { From 29708d930f75e88ee8db0ae2dc3fca7e71f7c78f Mon Sep 17 00:00:00 2001 From: hartman17 Date: Tue, 14 Jan 2020 14:02:55 +0000 Subject: [PATCH 174/257] adding WWT to adopters (#1994) * adding WWT as adopter Signed-off-by: hartman17 * logo adding logo file. Signed-off-by: hartman17 * adding logo to adopters.yml Signed-off-by: hartman17 --- website/data/adopters.yml | 3 +++ website/static/logos/wwt.png | Bin 0 -> 19270 bytes 2 files changed, 3 insertions(+) create mode 100644 website/static/logos/wwt.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index c84633a9d5..ce0e17cce6 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -72,3 +72,6 @@ adopters: - name: VEXXHOST, Inc. url: https://vexxhost.com logo: vexxhost.png +- name: World Wide Technology + url: https://www.wwt.com + logo: wwt.png diff --git a/website/static/logos/wwt.png b/website/static/logos/wwt.png new file mode 100644 index 0000000000000000000000000000000000000000..4b0d0adcc7a842a3c8474ae290c600823123e863 GIT binary patch literal 19270 zcmeEt_gj-$^LJcVeJqF-5Cm2fDGEpp9TgD~=>pP%BE?V=LJuKeN2M(t2?P-YrI$bw zAfQVVLPrQC1R+3ZNvH`B%6qf>PkeuPy)G|fa-W%dJ~L-#&di+m=B9>x$Api8Kp;M& zdv`2Bprb4h=#Sz59R$AF&g}mN{BzXzo}E7kB!7AT?}3mndO^UK7Xl0(1z5q|1A?FU zxq)uE!d=|{GV*!i;b!Uf#1)BXbJGSMN;H4)&|q(G5BT`M=L4*3*Zz-ukc^)^1iXrT zfTgJw=+voGfBp5>jT<*^-n?mUZvOc3V_#q2=g*)2{rBIWK7A@HEBpTadrwc#@87?d zmX@&Ou_+M9Xv^r%t%o5a%rWNYz82xb<`%LaK0i|V@LBr9X9AeCSEyB;UuUXy)++pdmFoRHe6v^%JDr4L^bE4=8)sLu$e}z#6S63Xt4|} z_I;!K{E_01csfa-p8W8`&R9OT=vn=^$Qf4t?S#J6MMSw4{Js$io~r|bl?js8>f@}y zm8I0BJp4ibJ=_53r^V^2CQy}@< zdipnTgRd zZ}rKBc{}0JF28*K72Ej3pmV2k4_je*)7N{m)9iW#_!VSk-oN;u0{Wf$HCr3s|KjI~ zX6>|%#zRR{y8cE7Z(h1}9|>a{RX1kYce&yc96{=NsZ#Wl`P$rJN!BhbN`(dazM6gD zcLm7`J34pMwXZbP_P%BN-pA&oDcL^I^NN=bSB$U8muutG5aj^3P4q^05==u%W^Py}whgo8*6hUx66$ACWsqmWI@E zcEpA7_!j!{N1^`%f!i|`Ud?_}$N}CfcTIlJ)|0Y9}Yct-8qrLm#57wSpW4zak>cpc2A@vY3h%g zv9;~~X2ge!1LBS1g4d?OA#h=kYfY&(*EOp7gNz5MJ;LaoB0Jm)u!7+v2DU((3zIPr zc9#Eig&TuA0@{Q~ck37-VE{?FmgWyg?&vj&3<%dt4j5}rW$cElZX?QF&J;@Ufu0#6 zSx4LbozI;~8tJ$zXq8ZS?;yyYK~i`nRDQu6pm6A<429B-8>cpl{{Hl9W}rQA}%UfNq3F}1@$OS6FZMjeCAB#9~Tn5<`!lN*mOV^U`^GI$GBxl z5;6NG?H;!B@!AEz{N=kRB)ddDB}iuU8ycH!~Daw zZaKFzxB=)?C}5(;dQ9&y03@=38R&0EyS zeTO`2p%Ar^NcI;WAHq?_2#+FoOK-?0K%oOT%o=s z&99krEBFDQd-DL2P5!IhAMQfDOQE~yEH;`2J3RsQG(ZHF^HUk&MCg-J9-@4e-53nG zy-)umU7xhYjy4j8h}>*$-hrDkp>|EB+W6kc#OSEZVDWt?<|o}PS8|1W$8al}-@Yz% z_MbnVzg%0vDoqo0fM}y`*DSsD5D@bA%ye0tqlA z=^JdoMSr=%ZKAs{h9bKi%|Yg$#(D45+DffR+bh)mxF{X%P1m)jsb@jw=8{*af7Gcr z#_J;fBQ{v8TW}_cvFj0xlWrpgmqxpm0y=IVFhj!p)Cs>=sA_Z`q-(Az(8Xwo`{TD~+%oXZx zUAvw)UQcY7jSO?Q0k$~!A+V&~e>5g)j2E`iny8z&lV2Dvg|7$C2D%$VKxJMj2X zxcHq};@>OO!q&|r^FJt!iXIBxJD&hF-rhyBhQsEH?$M6`loUd-(T71>DIHRMZ+fe~@<7Tif-=(b+4cd5<_?A(9l5z^8`XjI+G{6=Kd%rF-zQbAW znPL)^Rt5LvddNL;I^wVs7B!G}g}abF{IT6%Z{M*^SYjO0y$ z7n}c*=y?oq9qZInQQ$uPt?ttJD8LwH_CF-)jAJx$EN-nYTpk%5?-j(+X-RO|&{YBT&SH2ww@ap+| zkG_7%A`Ho5OPNf8ZW5Ci0&6r}+&3s&Wt!-7HRr(dl`wPc=ogPo!$}%Lzor|nrPr4Q z1gYhi3jMiUzHoWx7T_8jg@Nd|+2`W8ICsgab(#owJPq=Ffh>)4 zP0~K0_I!nE281Puc!f^PZbGC~CIEK(I!e4Bn`(>Y$TeW=A3!4Fdf|LdXSXiR^mQZj_v;4y5OC$ zM13!Dc(8Bo_&pTJw-t4c`+$&C5IQ=;=?)T>WgsjHY=M27d7Bd# zUpbM=far9L)6Gtb-k0u1_m8=%fNn^)`zx<_M%neGMw+^%GV1;24}Lf4CI1{QBl|mO z7qB>^`mMAU@wr0SKIGXJSX8}4%($+YCRd4m5C1Q1#>A<1fA%@$Oy2bfvvFRi`inIH zNnk7demh4;Tz@}J1nwX612WljtEMr6n>?Ob6 z=%03JAjc?4!cp?>(HEg?-;E+|LD2u^a)ddI6>9B1UGjK5f4x}fQtt(T@EWjx_XPBl zta5v2NYKb|X@JJ(c=4uJfxyz%i z5`EZd6zJv<8?45xLc&KVN_{ksW`O%prARtw861hSw}o63UM)gvNx6fQ5y z`~rbo&^1UX+ZbR*d6TOHw>SuL8Qb*h2SnrMC0ph$|9pQObf*?NO%ycGo#68X-AsrC z%Q8tn$1Q;H`us)Bt+=N|Pm2T|E=ht9@Qn8Yp90}zKbMl=WA}kTN_AGbr_CS4x19C3 z2KsxVD<0T8+@ zL7!p-1)XwxCrMB?kSpzTPLbb0To<=L2Mv3e`@K&P~ET333D=+Ql=%Lf~ zkMlhbKfVEE8E$zbkImYtHs2@}j_O#C>{76`)86?9J~@jSA#^&en3@Se;a=`=2{Zgs zlURZj>&Khgi1z>5Uwr>LOT|=vJtD}%nGeCS>=#-C_GF~DL8aYh-FsJ zeCO8jyA)vf9_1U41=QUTe(lRW`mMp4xRsPwG4onwMznkVx2eZP7{xzCDB{I83rZcH zX$W!wKK!uf$FBk2>+=;nkWEt)s6K;!Wl;1H0oj$G9io$v{!gC%dfJ70_LX(DFUtSp zaW;&Qi{nZ}=p_LLq02K7wg%P|)E{)1R>}TR^>UKtoMLzFsz|KOr=L#+0{z~h05pyW z&c76E^&0?Grf|eDcNuSFV~8tZf3LP%_Ot#q=j2VrxE22x{Uu&(GT3YX;ol|T#^xjj zWbBtjOvIvp$}46Csw{9b?um>#YpmT~k=U(kkNN7aPdy9BL7>(JUosF5UhRhfC!{!g z(Vx~Weq|O`{A!&>sh1H7dIt{4I<=S>dL^9p=XiWL{I7m)?S(hI#&KaA2Ethm^Y zVkg*V{)aH{wb4xfRugH@!BzVvd@ukvGdId3r@sGBj>Uncep=zS^Ik4+J{Xnbr%UHZ z6{Lrj1>Pk%CIxz?ig2%9h--#v1F8AdLc%*~Kr54CjupZBW+=H2Dp>x8dvx*9HE4aH z760HP*V=Z}qbve9&S~qFx0;DzVR?nSDA(yf^N|z@L9zxK7oEdvlel?0%-=WU`%_;r zg=@dWb>{s)yuSBLWsLehbgK6Q2B^PKOU~tEZSXD)eoHmSS=UW|>KuYQaye{T_W3 z41xfB?zSaXR4(RJy^CJ`kIx$sAi83<$G2yJ&9z%@nweCbUR(C|BvRu)CZ4vFF-mbZ z`fi=FN}vBTfmGJ;jec6?w(~aN9b!uQAk3UcGv79EY=4OK>55WsBE7qW-;!A~S?%?F z1SUpxME=LA8KvAL5!1Tauy3J^G0}ez^U9h*FLaBlyMh9Aq9M6c1h7$N(%6yMLFwhX zNB{8x_bj#hm!5+*MH2^3*2LUK4`~{ONYXY2W?QUbTodgNyB^#EJ}u75I`zf(qv zHGOiA4{x`>kv5vYxBHJz<&%zlIS1ViighmzqUIkv^v=g}&x!i<(XlvGB~Gf~&$z*> zK0VxML*sv~R=Tb6(if88X2Yt{B{GLbV{QG9mDqwF9KM$Dc3CLpA39B%Sw4j6`7x4@ z?tbu}p=TdnQ%@0Ko9yp!H|`PNOchq;@fK3L0HTlUQWo_N6=*C+F|yFD3-BA^>HGiZ zN-FAz2MmPweWjVH@3w#pX+R-ImpAkxula^pqh% zKiy!zwa7()c|iFqB;G(c@Z+a%ZoDt2sooNl;Nr8+dy=d#ClbWOC3oDujE4#$L2BRN zmID7e75TBuOWLv1nG!LYwXdfiH$-n;w~Zgb@QP;ql?qcZ5psogcqTIhva{1$cyeTO zZnE2Kl(n}_@Kn<^*(*1Dba0d0Tc8$yVEzPu=Y{Tqb-Yd>X@{dX5Z32L6+vsAi3sO# zwz{8|e=C?XqMQg2;_eQwm>4?frgs$%G*G&0>>n5s^}ZHpj0H!am%{qSpv#QIYZ}isdXn^0biZEfKMshDxXdd?(suwX{C%ubl`B@(}>GdIpCbojOyR_+7Y}v zk?@aY7Z+0%YYfigzk6kD3#IgWBXQ($d5=0*iW6+`UW93A+@d;2Y76jE zP13Cq*{*(*t!2?Og;`G*!ga(0Pjg$M*2{URJE4e*pA-w~LMXX)b)dX^w0MS|A^va& zSKOc`#N;ly616Ib$W$I~WN|e-e1Tk5GTpu-vo{VN;860)2>B^~n=EJF_!hDy4HHz6 z12%qil4Iq*3G|(|LJ48wFx7mzpL9zRyt_4i%!#?*aSBO z1Si^24_`kc;r3Whe1Txo8Y#^!M34z!h+6U)rfJef%{vsm06{MNB`2;>y-6{XW_8ZX zceNbUJDMSrgY1+aDowEJ^d5Nl1?TiO*n>4aa*G_eXeFr1bJvRVT5-qF|5;E@Y2EAT zne4O<`qzz#2l1GP1fhS{6kcZbm3A|PV(0>!MV4ZXN=|RaZ&qR5w+q;Sc7nR*nA>&P z`MY8pnjcU(8?7#Tq0eGZjLLmJj97@XSEa_G_hkKl-v zV^$8ngEVOkxVW{{c}0qKNH6HTR)z|sSyi0p`74mR@O662tw`t8B}&e_tMEii`ufx7 zCZRo2V0~iTQz(!_0wakD?h$C4`IQ(6nawWQu2 zPJ>blKZ{2>iGI`?Z=vSJorTTzPtqi`H_F>Tpy(U@?qloP4{Hgj71N4mU*uS?F??Fu z5O6Ww=YfU|GAx6zMcsMJ3$1%w5%L`+le8;eray>JM5*fqCHhhe?fyK|;WJ>4O>Mt$ z-T_;9?=tn%9fczfqR7}*r97E51Kqc7xH{9mo&$q;bZZ~Por<}`Cu8jdIH*zPWus^Q zu39#VPlHxpL$tp%#{F!Gzy{~65%Bjkpd8VLi@Da+sQ-PBKQPU{CO5Y#@V+%*K*Ai{ zM3S9@(fm{S>yPxUKb38_- z(+B(vie%r>=n>PT7US}ga!;{&Y;)|NzKz=TU;j(#XHOWy1dt3ZozUUdfObXYKiV#f zcJc*{DcCy(pXbgT713bJvpO&5*W|T^)74!d?nz3xGosw9M>ChNO0rC97N@&~xfRu6 z;5jAcj@D@*!~-k$4$3k@fvI%A9@@FF^9|TE?cT)OXKRjT)r!0kyIa;zcvD5B3^1~~EH}42(^UwJN~W&t5XiXT#TAY4dspQ=jJzfxBapNJwHVPy?H_%K zL1TW?JGp_@4L;acoe-f-fd4%<(bR1Y_6l zIfz4J7Gfu9DvFSfnhz+|!u$wmHN}uk2X{thhr7#?K2&y2WVW=BmUUNi=j7RLEhLLp z2Q2!~j^E}*y}b9wuSg*f3W!2{W?Tk2A6HcsPN$q7M`_!NFXggX{BJZs*ztkD| zvJc29vzE78&OudL%AJK@GJC6YkAbrm9_vE;-NoRlad>o<$um!_EUFz`D+;ArX+!Uc zK?ZKed00$DU=jig21&~>5kanf1{hOPf~_M^Hd7c!uvx40Wd)zw_*Ob(z5rjq!v(L& zM#rMY@QyJSg-}g!S%W1cmNukVbVU?)eqBA~xO87gcs1aduXoakFag*}&X2n}))-{K zclySoe@Y4ocp~eLDHU@!#v(d4f}%}*A-1#T27Ep14&O=lAC%KAo>yg69hD|;I8RzI!0=e#J_UhPj_&8-;=zhd}#^|sVGyv{;8;ts*u6v>hVj^7$| z#Ax%{AN};k2>S1PJ?cZxifZ`(c>nb1duEv7rsOr&!1Wr)+t|U`jXl!)_T7FpgaWEX zH6TflF}^9*b1bmoNh=)GiFlkdPtDQHEx0w9q zBcB?v0=WPj8!98dle*vJl28n7@4~dhC`f+}Q(tFH`&WC%DEAL?T1vr$TRM~(X1gh= zfGB#p6;qIxe6pGrm3vjUz!>73b(K}6eWDhY!k%s-Sx_8ku#oMBrw{Tq@LC^GJWfA$ z*s8k$Qc`&q=jg;L-gEp=yYVH;q@l}Ees6>H9LK!`oC$jQuJt{GRkd_-F_@aUXy~0Z zYQRNo9wO5hjDQD@c~CK}Q&EAPC&$IabNE~<12F`OHD%p6zm8R=pbg*8Bt~P^N=;93 zZSpma&oJ|KE(?@2HIq#3OLF{kMgcVDH5Kc>Xu~r_<_7*A$8h#j@YOt0O=Ny|y6v~3 zdEqw(clxVly*J6r2jZ++NNs_o2>8WSyV3Nw5$EUif{iYN6rZ@FJ`)6wofwk+{ z$aRmq)Z=jEv#glfi`Td3R%-m@g1?o6o8HnWF%mM<$AFm z1s;-!aR8Uz%OQ^0s|9;S)#O3X$4g3;n~b{Kb>{U$t$jE$2gUNEQ9sH*j=5J3W3RIY zrt%9mx!v=y<r>=OHR!JZB6xRU-Ruk}0>Walm3V0CD zvWalIx#}sRaZd20^n#}t!l>&EoyIBU%d{r@-2QN447D>BcD-P|WT&`E1u6@!p4L># z6cH|?hS$3EMZWOjEL@qZ_LG@ha%z|++VWR|>j1$#9B1^kOq&1Z!uF6=hkfXvc8!`5OI^o&N7C~681oVytMs5$zz^+}Yv15sXjmT48wk)kJAI#vtGg?`jY1sAfLMDQ&(P9!oIXne)q(TKodndnT?` zF(mXPxIc+A-Jn%Cqgz#?uD(rpIJ=CvE=R*~J^^r#hibUjK3=sMs6I0~)E+b+{-vZ1>=Ju-(| z74~#}Vh2V|xyuWArefUrJn^`CeN}gI@aYSztp0L&woi_HCc5v{+Z;Nr+!?|PstR;e z8WE46+wIJ$o5i4NaVdq_l4*Ck&1|7eve{n!%y{e>`;TwJ8Y6ae>m*{N-aMG9CyjMq zayV2QWy0L}r+^p_g+1|PPuk|>nvhK86sYoy{)h&b^qwxt_%HDY(!ggP0e_NXJ34|W z3iLmY6>@rs?wHjq)`Nvc&R?BW`HF0QG}aolwADRDu;p_4sB&2bhtkZ|MGPG^4G%)m z&_(qiA$e%Kmh2j(;R>8K>Fy%Derq&H&em$VO}iZZ-r=s68R6Q6DDd`_odT3?uzNTa zDA!b!URL9$-D7)M{quR5eYNxH(b_fZwZeDnhWIl!$0WNZi>xr4NCrY-+b?J9D#>%Z z78;^0DaRVv-KgOjnoI&;wLJ0~zpomVVW_1~HKwjWpQ*z{b*d>U0V@2?nt@NV`_n3G zby)+M5b`^n+WRq%d>&I*P-KzLu} zyE6h%6}3P0uKyCE;L-gphs2^Q4e5F4TXtdbO(2dfkla6eP-R$=)Ow zp`F#dy#Fa|I-2oV+)5x)Sd#FaNH`Ou_=uK=`SSlEwQTLKhz1#)C zqQ2bAN%r}qoS{~<@oe#qZH#EEY!0y`Q)dJq7HIYAqBn!lmia^<;mJbP1-(#JUk_Am z{p|8pPYC}Gb;00FR2o5W-824OV&0B7%##`w`o^RwMIqhnC)(x1$RU2ac@WFLCiu?@AH!*CFM^?l%i@%H~Is#Q$C7|Ao0;&ybA32^d zq{sRmnv&%1-r43Vo;Gxh#C|M*J!|A7b8f7ww4@)5> z-#?Yz^AlQu_5+JUN`0{eyjg5!r$hbxGo0_ajaw#ea=ax;MU#e$jortGAdHK@5b9Un z(jgGCYRv!MPYfiN(?2!9On>pJ7eWiSqV?7VR1D?^s zm<_w8PS`&?$d3LiHX6)j6_ql9ugi{s7rE{)s?q&DpWwnlqgMjs-fX4GLeE=%GNZoU zHCjbh#%2A~d_u*zdT*A@iEzG&3x!@YZrSP3)MLdY2K%Vv-?V(P`H^>N=)?oR^cPzx zg~#{`Rh{3i4iWGl#LhWqeyw;PwYC%uWn1sM*I%C|LDAx+X#XR%UhizI8^=3^9b(+> z3O$?E8k`{DJGZ8wWs%=1TF&|b#oX`RGJPpy_n+R$}q` z)2Cm96eNO=t0A_mip@NYf4tb%j;8b|7?TjsA? zt&Hv}3);A>$0`=7Eksy-g%%7v`m&2N7x&LhW-xvb@NMdZ#OQZJXsUz9>n@{47$i4~ z@xnJn@s=hz78NDnV>u>0cBQhMH^Ki30Z$w_@kbtzpu$zKPwN9^RI~M8 zte8qAuBP(EfD7CfJFVOaw9V++t;*iDG^17i^o%PkRjIX`R1vomLTY(A)&a^lIvw9w z*7jOIo0Ra4#6?G=5`R_(q7wp39LuykfKEl^gdjAm&Z(()L{0U1VM|G%(`d&7VTVrH z@~?{OQWc(PuBayi{}EHiQe%m-&!q%9bF#iX!Hndnny?rL(GQY`uXPS({rh`PnqJVm^ZSHB913vKdliuxo%v*Pa__KfMCbu@k*626j4f+%eopph}*e^aq#JS)sZ3c4rTrI z_h;Iqrj(KCK`D+}!}N$Gqx$P|yX;D;!c$Yiu-r4>2C761P<#T?$fv5%8fPEYr+F_` zn;3m>OkGNG_%=hc=V)ZsnRG#J!C)4~b8u}HbGfsqy3|f;yzdC7<*HYV)0j}x)gHH! z0{6n$xLd&p1vuH`<16=6AY`f?nKtXc5Bn87wrmCQcdjl^XEdUOTN-1tjnK7#)A0{m^B=8$=ku*-Eyn?!ti;=WvsNs_RqGbnx+)sA9O)y9m4h(k zA566X<7xT#Pzo#H+4bzd-e|x1vRmyR5b(g|%zP%ds#U?7TctjyJ?I`f zk`l##Ko!1dCK*u1R*GOsC;@66y-GMS+Tu+SAr+%xdX_sBM`Yvn(yLgY z2_)tY|3Qk2bRj8BN&CdnC@~U~o#pm1Pv;N>$Tg#lL2Fwd!P}H|xqEG>%yUrS zgr&O?D*fovG?AQhS-bz;UDv?e^|V96beou&YT_+jFX||RqrjY`&Ci-;7;ew6$ET@x zZ#umgtJAWejsk5fNF01XC|H~;fM|?`KHUk7WX@JTw&BZCX*eVk4$tD~?t6f2Z?_~y zS8TP4b^occaREaYm-El6V;j1p{Aj;xQVHNT7{lxocg8UCK*2kOZ?TK%Ze2R)$KWFn zv?V;_G&cyNz0*n(aHy}XXN7(lcsxm~#}g>BP&GJbxnkJKz0q3IwGr)OC8p(iY%g?R ziv-;q-`O&+QEK;(tNx?#-Ay%(K8;fS9rLKWl~-j=0Zdk$sknJvmT2jx(K{*w{g>Nd ziHYQjNeJu7kP5rQO)>R>KGo}@ZQI|Z$bAdkWVR|(iky^hEd0K!8r*)_r*phI`}}Ac z`O87}$QP2wu(Km9{w^?J(3)~OPEFB2%+O_0J}Jungvo3B=G7L`yP)uXG*xTl*5~c}`b4p{ogv zMl(7~vd}02K93lpV@(e*R6CF0aCbS&H$Yj&z(DQbi3d5X^>80h>kp0h0?fekX{+ki zR5A-v3g8;cd~NiLMRmJa91W{)cbX@7n;ICR3bA5*DFWOXq*b61*F!Fm4Zym*cQH`- zPbFJlFADL{c~u6?*TAK0ZbWitm=NCGxiWaQu^Xw-pv^S~LOELJ#6PWn#P*cCKfW;V zc6;QO?T;(?x-QoC?J?@NNt$M5)W-}7mWfO3pahI-J@#fu_E+IsLq2?`VQ}b&^r*@v zQVG7UhNZ4|66n&b8vE;WFJ%{Z*2jGld3)SC6Rsr8%4@9tMv_m>lvn$Gm>hnQoKKeP zlP=We-g{aZvHhdnW)!)N>{IvUYb?XJZ!c(V%%7M+LUddsj`IGr+5B05T))E!)Sv; zt^eG1(CUH_U;03j2mbLsOQ(Hh)m+4ozbJS6_0gghOKW#HH?G8m>giVc9vhMjM8)`H zdDH=LVGrRqfV8^8_tzfES7&q1SDpd5f|0MXf1HP6Vvjp^3v$(lqv|b9i`)R% z`4m__e)$gtv;V$G9YM3MY$>V8tCPB(o%2fn^0u+MWZ2)|pBTXCv;&H52ylNyv z(5Yq6uxyC~fA>u6%VH+rc*Y#cR%Yx+fUygaKwOM`)_2q+a zK*cK6IoARh+tmagemDVAeE;CqkI^>SjbN!L)Iw1^aSRINK%SWVb^iCp()v7?piJyt zPk#2*H6p1<@qRR_zvFd62+1^oecIkhbEk;}9UMNd{9J%Kf^;XV@{iw!zw(G!`wi)T zGAiBxrUpB38~K76#`f_oV~g-mB?8X&CXyu~bkH+>c1dMPx8|d;$C$+4&{!5w&0Eac zk#-5;k^v*~j;rD%bz_M_XU&O|6{O4JkRgC&Mv zRq@%pvdOu;_PY8&GbvqheoT5HHYlN@3MprM-lEYa7|Lp$!_V>o{+YQ+P)aq%i(%;` zDBfB8?Hsb(&?6PkY2et@E^`9nD`DQkr_D_TxSrf@b)9|QKQil$9>}~`&iOG`caI&J zr`X^YPaC$$zi|3N3@X!SvwAtK=uG7g*LMhbihrRJEmEXCs#_BqtnVO`kP5^o+n_>VnkKZP^?4tgX>Rn`~1#vFIDU?&Brdp6qJXsn;( zda_CBU9WtaG%ckHj6I$T(|py>jYwYwm#S#F!(6IA}tMsYYOM zCfYvpT{w%;%YO@V=Q%&G1p0%8zN!liOFVCXY8N!YghkO{5%zep_XJHkuhhRSvG=}@00V9ur*DF^fHJVVm6q*w#C zjbh!X{+;!sbecN89LvbtYJ!Ewt3B-fo3ksz6?mWN1Cw#%BH*;nlv4f6j+ms;!nPZ9 z$d0*h=cZO{ZRq9dYBf2g*XJ2m!P??=%J*GHgr-kPCtTaRHVua-9g-W{3}^>VXgViG zTOq5ot6t0qbt%)c4L%z8g>O?OIeNp0((os1o=G9AW4f$EV$BuZ_4qi4PolKWb&qDw zPumB<fy5?Vxgz3=ue`MuM&ukn!en5R|8=@MEq?DWrb?Mx-u>= zg;Ys;$12Pk`OzCD=i})cj6rb~@n1dxRG}TCDTP@Fovk$Ozo;GTl%uF;PZykV z=wr^8rd}+)e3wz6rhE`E+xDFq9QQi!6)*{A0ZegUfiok%dOyxwwOwd#(7&0u=_#IN z22|38&knkCo{ESUM9%)jGV#aoOdz*RS72C`lh)g(eBHApvQ1n)*8M%S3MK_W8Q1p1 zJ|E9#wNVdKG%1h0)l8b{UG-UK3r2gybhesZ_O@`S^@I`gfNlINM8Rml#r;$Hz}qV( zfU}IN)Z@??jgqsK+W6%X1a?F-a@Y4_Xv$Akw@36Lwrf5k3e_3h%nMY_GW8gc{jA9R zi>0df!EJE8_kJAeg(K0e)_qBtlptMrcF}Hoci=~gSBhYh<~!7cM{zJh@6Yi%bzlbN zt<%34$@-g6RNywB=07_*xKI5}B?#hYCGQF%N)Z9LY!;4Kakb*Hsyo=McEkbmi}=+< zT0f|KPk9oubdsT&b|$Y^(0e6T)P}k+vo0p-W`6nSbGBC=#U-|odh0(Cz}=27z@~G%BTb}1XfEuM%L4DUwV zU8+~Z^g`1{nxF8=>#RV;>bkLYz|>$+kKaJut}8N@AQ*nD21vCmfh#9h^_V1G$JpI= ze?z3#9wz^<6raOdFR$8ZTmh0ce}{1kstBWckhcw!@Bznhfs?UMejO+Cgwxc&DUMT^ zV|7EoRx~Pyqnx!`=l&^*SD&tPV+1IrbS4I$a0O0`hCm;6F|KxJ*Vl-~3362X$ZVB` zpSRv-m`Q7GbVCkPo_t%eba;xVvE-F~ zCTMeXnya%Z)xkYHeBhR{I1vK|gkbu)4s`!xMzv|)Zg1dy5+T4pDx`@Az6Y@ zV415_5Ln8T0xuW}%vq4v+VpQBZPx{dkr_^3NS9hjYMrgUV;{?^R?mJRLw%|FCT26n+(hd%N=*0nZ%HTwDQ0 zdtj!Tbv%mzV3sA+6g7-c0J8^tHVbla;Gcw4y)4w)8xJD~Ej^17;|t>$V6gX^KQW^} z@+2?<#bZt=tv|$Sbvv>j)<$2S%yjHW+wKe?Cn5K5{l?300X*V0a%;Lo&UcJ<$^dDS zO#qb~tx?FrN~MT$SE|Ew?$>XeDaGDuA-xrBa-&|)^YL;2<%oM088=l?Cs2w|3-&u+ zv(BbJ0y2znUI^hK9HntJr(@(L62ogk++42kV6Uaq$-e|JGj_-L$@aRp$w{ z(5Ip@+^f(1%i_$Yt|n8mnKXr%K<8iUacCkr^gHD{UDfj z%dbaXY)6z2$k1s@{-)JC$C$Ze7gfhOdv_AgabYnw`LodOQ{2y?tK!w#xI883>p~U; zT82A*8*3K4UFXweECDwB@&D7$Bb;fFKo*)iD9;YnJtMO74Z#7vkwh(wXxefHFMqxysW^|J_jx}-EQZhfX2 zxo4z!T}Dh~RFj3q9Cc*XP*;BQS*=bhFA}&uaGM__4R-y74c<+b19K}+&VJYLAvg+g zL=NJUP1z7gn))22Qjdz*)zQL=o!zeNRbjKk-^Z+r{)U}b{TBG(&cUGVBY)bUItM3o zLlc&X)8fS$W7-lhb1!)rEa*GbR6W{=w!1@AHcr=?YrV^_6HJUCih2;J0jQ_-oI z#w^O|tw2X_Ek^b;J3BoUHH@G3=#y2uVo}Hai6_3VHpVPZ<%!YZ;Eiz@v=9!i?4LZ| zT+es(`Yv_w>ytaXx)qgH7l-r=J!T@pw_HGsPKASRV?LW%yzH$#4Gqz;-!o6+m78yT zO}Xa3wQ0lDx2bw^Zt5`5VspeE{|@R{$@)Z8Of{Z^vgs!GQ9DVDKg{c8^;pb~Y0GL< z7C+e74Bma{lP_j*YpI;**v^?W+6dyvZ;#4t{DoqB zT~ATd`Spprq4PfvI##r)(Ay#l>nPIU$|LxYHjRuSU?^nc$K@m9zhC+&(Oh-9B1C~M zn^)z;Yu4BhE?-;lH_&BiIe-t`tz3;AoDRLKZ4k1w?(2`X!5;S|7A#C<*QDDfv;7); z*wW@Nr6NU4kn-QB(X2lokTZEDse>fJU2ZoH#dwZ?W za95cI+OEVIH1VwA3))z?bH_Dr_IfgK{T6u2$%!^9<`4xgz-;q%Pj`8y91Wcn+u1M4 zmwtFJwPRoclJ8%s8JuSvs6|wj?zFa!zp8e$j4yw9Wi5e`VgZz$6 z^*ws+_|`~Msww|-vWh+9HEE}4(@KPnrR4W!8Gfk)GlSm8KLoV-C7r1$Hgdd42O1}K zUWvf1L1sF_lN*8ekhe)z%c(s^8Gz;L99#eT8YdX`72#af=p5ny7UWpas~!3p-Vlv^ zci)FPYQpr{TF324#Jtn!r%0MA9lYV2Xx(&bJX|%3hf62mOYd@2&j-`oz_z{{ zopBz(O~D$zOXkqvQ zOFfWjK1GU6ip`X>-hV76KVtQrw!Arzb{>7|u@5)MbzaU$GA3gk7)C2y5d+-_F*_We zv;V`?{ADBbgYd6oqu3Q1O6q^WPwei!cZq9?m9!7{AzHUU&$G2q3GMgI8}@HW{S25{ zp-R0PYG;e7>w}*EV37O;psx8+z$GluwA8maxr%=O{m&mE*utJFYrpx0{R_PZo=<3X zzcL+3At({k<^}3!7$)se4uHRb_NI!Uf1!y##BXHJZ zodJU1molrh@r5gc^BD z49zL6yfo8F%}g;f-Mpb&6oV$&3dxQ)Dhza~>9po3H6#Vi1g%+Pq#~KOS@U>F>UoaX z-_VD??d4_9@ACX!&S7WIVTDe-7iCQyV@Ru6q2n`6PqlD-iHkN&HI|#>C#wmkY{=() zEq&eUqMZg#1i8R_TNwN@k9Ah8j0=TLTQMzHt5fXHbGH7VW}`^V43$w@MA^r3mNr*g z60Rl~{^sNF+IAX=U%jU+JTox>nAZ?9U31{ZUh~3`wWeOJr8`{L65D7I39{ag#CwES_rWb(p!P*(~^5m8) zkAaVIds5iEME9;|2qxCp*swx6#^JEIRd8o?Up$g1k*me-wkwDuAe&G7jrBt>BUgF} zKS?G$Jb`pZoOE-ufjovvZjhHOlUEe}4BRsQp4zq{kw2cqS=!aBqM@llp4rQv#q(L^ zuRK*Lah!p~Qj*twDn1xJ>tr!VOscBH6;Zj(ERm+S@D#8hmevsd=+ z$^df{2+2PgvM#htKRA2_MYPk}x`>K<%~gmXYL_OKJ!nvcFf6NmWS-A_&DTS+rmOc@ z5*N7nMcseKgozd6AN)keM02u1~n6WHT zPEfCWmqy~=t5Hs6=cvYXn(u2!Pcbct(9!x@7U3!w-QN0ch=c`GLn3kO7AfP_5UvdZ zJ5zY&0VdGnN~Fza<3Vpu2$6P4l@k(@-@x$xWT1;_e-3PM6$W-^?4SiHl_OD`Ua3Hp z#BW64ueTZMb3! Date: Tue, 14 Jan 2020 15:11:16 +0100 Subject: [PATCH 175/257] *: support Alertmanager API v2 (#1982) * *: support Alertmanager API v2 Signed-off-by: Simon Pasquier * Address Giedrius comments Signed-off-by: Simon Pasquier --- CHANGELOG.md | 1 + Makefile | 2 +- cmd/thanos/rule.go | 2 +- docs/components/rule.md | 3 ++ go.mod | 2 + go.sum | 66 +++++++++++++++++++++++++++++++++ pkg/alert/alert.go | 76 ++++++++++++++++++++++++++++++++------ pkg/alert/alert_test.go | 6 +-- pkg/alert/config.go | 34 ++++++++++++++++- pkg/alert/config_test.go | 33 +++++++++++++++++ pkg/testutil/prometheus.go | 2 +- test/e2e/rule_test.go | 21 +++++++---- 12 files changed, 220 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 006087f904..a7e187c983 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags - [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction - [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`. +- [#1982](https://github.com/thanos-io/thanos/pull/1982) Ruler: Add support for Alertmanager v2 API endpoints. ## [v0.10.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0) - 2020.01.13 diff --git a/Makefile b/Makefile index a29e8a16e6..cbdf5ef39a 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ ME ?= $(shell whoami) PROM_VERSIONS ?= v2.4.3 v2.5.0 v2.8.1 v2.9.2 v2.13.0 PROMS ?= $(GOBIN)/prometheus-v2.4.3 $(GOBIN)/prometheus-v2.5.0 $(GOBIN)/prometheus-v2.8.1 $(GOBIN)/prometheus-v2.9.2 $(GOBIN)/prometheus-v2.13.0 -ALERTMANAGER_VERSION ?= v0.15.2 +ALERTMANAGER_VERSION ?= v0.20.0 ALERTMANAGER ?= $(GOBIN)/alertmanager-$(ALERTMANAGER_VERSION) MINIO_SERVER_VERSION ?= RELEASE.2018-10-06T00-15-16Z diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 037bda25bc..d734da03d3 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -381,7 +381,7 @@ func runRule( // Discover and resolve Alertmanager addresses. addDiscoveryGroups(g, amClient, alertmgrsDNSSDInterval) - alertmgrs = append(alertmgrs, alert.NewAlertmanager(logger, amClient, time.Duration(cfg.Timeout))) + alertmgrs = append(alertmgrs, alert.NewAlertmanager(logger, amClient, time.Duration(cfg.Timeout), cfg.APIVersion)) } // Run rule evaluation and alert notifications. diff --git a/docs/components/rule.md b/docs/components/rule.md index 739bc60b50..3f7cd556fc 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -416,8 +416,11 @@ alertmanagers: scheme: http path_prefix: "" timeout: 10s + api_version: v1 ``` +Supported values for `api_version` are `v1` or `v2`. + ### Query API The `--query.config` and `--query.config-file` flags allow specifying multiple query endpoints. Those entries are treated as a single HA group. This means that query failure is claimed only if the Ruler fails to query all instances. diff --git a/go.mod b/go.mod index aa141722b1..4f9bf1c077 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 github.com/go-kit/kit v0.9.0 + github.com/go-openapi/strfmt v0.19.2 github.com/gogo/protobuf v1.3.1 github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect github.com/golang/snappy v0.0.1 @@ -66,6 +67,7 @@ require ( github.com/opentracing/basictracer-go v1.0.0 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 + github.com/prometheus/alertmanager v0.20.0 github.com/prometheus/client_golang v1.2.1 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 github.com/prometheus/common v0.7.0 diff --git a/go.sum b/go.sum index 640a65ba00..d122df9d2c 100644 --- a/go.sum +++ b/go.sum @@ -59,7 +59,11 @@ github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMo github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.6 h1:U68crOE3y3MPttCMQGywZOLrTeF5HHJ3/vDBCJn9/bA= github.com/OneOfOne/xxhash v1.2.6/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -82,6 +86,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -115,6 +120,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -156,26 +162,65 @@ github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80n github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.17.2 h1:eYp14J1o8TTSCzndHBtsNuckikV1PfZOSnx4BcBeu0c= github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2 h1:ophLETFestFZHk3ji7niPEL4d466QjW+0Tdg5VyDq7E= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.17.2 h1:3ekBy41gar/iJi2KSh/au/PrC2vpLr85upF/UZmm3W0= github.com/go-openapi/jsonpointer v0.17.2/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.17.2 h1:lF3z7AH8dd0IKXc1zEBi1dj0B4XgVb5cVjn39dCK3Ls= github.com/go-openapi/jsonreference v0.17.2/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.17.2 h1:tEXYu6Xc0pevpzzQx5ghrMN9F7IVpN/+u4iD3rkYE5o= github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2 h1:rf5ArTHmIJxyV5Oiks+Su0mUens1+AjpkPoWr5xFRcI= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.18.0 h1:ddoL4Uo/729XbNAS9UIsG7Oqa8R8l2edBe6Pq/i8AHM= github.com/go-openapi/runtime v0.18.0/go.mod h1:uI6pHuxWYTy94zZxgcwJkUWa9wbIlhteGfloI10GD4U= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4 h1:csnOgcgAiuGoM/Po7PEpKDoNulCcF3FGbSnbHfxgjMI= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.17.2 h1:eb2NbuCnoe8cWAxhtK6CfMWUYmiFEZJ9Hx3Z2WRwJ5M= github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2 h1:SStNd1jRcYtfKCN7R0laGNs80WYYvn5CbBjM2sOmCrE= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.17.2/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.2 h1:clPGfBnJohokno0e+d7hs6Yocrzjlgz6EsQSDncCRnE= github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.17.2 h1:K/ycE/XTUDFltNHSO32cGRUhrVGJD64o8WgAIZNyc3k= github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/validate v0.17.2 h1:lwFfiS4sv5DvOrsYDsYq4N7UU8ghXiYtPJ+VcQnC3Xg= github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2 h1:ky5l57HjyVRrsJfd2+Ro5Z9PjGuKbsmftwyMtk8H7js= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -222,6 +267,7 @@ github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OI github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -287,6 +333,7 @@ github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM= github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= @@ -326,6 +373,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= @@ -337,7 +385,11 @@ github.com/lightstep/lightstep-tracer-go v0.18.0 h1:fAazJekOWnfBeQYwk9jEgIWWKmBx github.com/lightstep/lightstep-tracer-go v0.18.0/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lovoo/gcloud-opentracing v0.3.0 h1:nAeKG70rIsog0TelcEtt6KU0Y1s5qXtsDLnHp0urPLU= github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= @@ -418,7 +470,10 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/alertmanager v0.18.0 h1:sPppYFge7kdf9O96KIh3fd093D1xN8JxIp03wW6yAEE= github.com/prometheus/alertmanager v0.18.0/go.mod h1:WcxHBl40VSPuOaqWae6l6HpnEOVRIycEJ7i9iYkadEE= +github.com/prometheus/alertmanager v0.20.0 h1:PBMNY7oyIvYMBBIag35/C0hO7xn8+35p4V5rNAph5N8= +github.com/prometheus/alertmanager v0.20.0/go.mod h1:9g2i48FAyZW6BtbsnvHtMHQXl2aVtrORKwKVCQ+nbrg= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= @@ -492,6 +547,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber/jaeger-client-go v2.20.1+incompatible h1:HgqpYBng0n7tLJIlyT4kPCIv5XgCsF+kai1NnnrJzEU= @@ -507,6 +563,7 @@ go.elastic.co/apm/module/apmot v1.5.0 h1:rPyHRI6Ooqjwny67au6e2eIxLZshqd7bJfAUpdg go.elastic.co/apm/module/apmot v1.5.0/go.mod h1:d2KYwhJParTpyw2WnTNy8geNlHKKFX+4oK3YLlsesWE= go.elastic.co/fastjson v1.0.0 h1:ooXV/ABvf+tBul26jcVViPT3sBir0PvXgibYB1IQQzg= go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs= +go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -526,9 +583,12 @@ golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -565,6 +625,7 @@ golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -601,6 +662,7 @@ golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -608,6 +670,7 @@ golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -645,8 +708,11 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190813034749-528a2984e271/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go index 18e57d6d81..82b9602d7a 100644 --- a/pkg/alert/alert.go +++ b/pkg/alert/alert.go @@ -16,7 +16,9 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/go-openapi/strfmt" "github.com/pkg/errors" + "github.com/prometheus/alertmanager/api/v2/models" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" @@ -26,7 +28,6 @@ import ( const ( defaultAlertmanagerPort = 9093 - alertPushEndpoint = "/api/v1/alerts" contentTypeJSON = "application/json" ) @@ -255,6 +256,7 @@ func (q *Queue) Push(alerts []*Alert) { type Sender struct { logger log.Logger alertmanagers []*Alertmanager + versions []APIVersion sent *prometheus.CounterVec errs *prometheus.CounterVec @@ -272,9 +274,20 @@ func NewSender( if logger == nil { logger = log.NewNopLogger() } + var ( + versions []APIVersion + versionPresent map[APIVersion]struct{} + ) + for _, am := range alertmanagers { + if _, found := versionPresent[am.version]; found { + continue + } + versions = append(versions, am.version) + } s := &Sender{ logger: logger, alertmanagers: alertmanagers, + versions: versions, sent: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "thanos_alert_sender_alerts_sent_total", @@ -302,6 +315,15 @@ func NewSender( return s } +func toAPILabels(labels labels.Labels) models.LabelSet { + apiLabels := make(models.LabelSet, len(labels)) + for _, label := range labels { + apiLabels[label.Name] = label.Value + } + + return apiLabels +} + // Send an alert batch to all given Alertmanager clients. // TODO(bwplotka): https://github.com/thanos-io/thanos/issues/660. func (s *Sender) Send(ctx context.Context, alerts []*Alert) { @@ -310,10 +332,38 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { if len(alerts) == 0 { return } - b, err := json.Marshal(alerts) - if err != nil { - level.Warn(s.logger).Log("msg", "sending alerts failed", "err", err) - return + + payload := make(map[APIVersion][]byte) + for _, version := range s.versions { + var ( + b []byte + err error + ) + switch version { + case APIv1: + if b, err = json.Marshal(alerts); err != nil { + level.Warn(s.logger).Log("msg", "encoding alerts for v1 API failed", "err", err) + return + } + case APIv2: + apiAlerts := make(models.PostableAlerts, 0, len(alerts)) + for _, a := range alerts { + apiAlerts = append(apiAlerts, &models.PostableAlert{ + Annotations: toAPILabels(a.Annotations), + EndsAt: strfmt.DateTime(a.EndsAt), + StartsAt: strfmt.DateTime(a.StartsAt), + Alert: models.Alert{ + GeneratorURL: strfmt.URI(a.GeneratorURL), + Labels: toAPILabels(a.Labels), + }, + }) + } + if b, err = json.Marshal(apiAlerts); err != nil { + level.Warn(s.logger).Log("msg", "encoding alerts for v2 API failed", "err", err) + return + } + } + payload[version] = b } var ( @@ -323,18 +373,19 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { for _, am := range s.alertmanagers { for _, u := range am.dispatcher.Endpoints() { wg.Add(1) - go func(am *Alertmanager, u *url.URL) { + go func(am *Alertmanager, u url.URL) { defer wg.Done() level.Debug(s.logger).Log("msg", "sending alerts", "alertmanager", u.Host, "numAlerts", len(alerts)) start := time.Now() + u.Path = path.Join(u.Path, fmt.Sprintf("/api/%s/alerts", string(am.version))) span, ctx := tracing.StartSpan(ctx, "post_alerts HTTP[client]") defer span.Finish() - if err := am.postAlerts(ctx, *u, bytes.NewReader(b)); err != nil { + if err := am.postAlerts(ctx, u, bytes.NewReader(payload[am.version])); err != nil { level.Warn(s.logger).Log( "msg", "sending alerts failed", "alertmanager", u.Host, - "numAlerts", len(alerts), + "alerts", string(payload[am.version]), "err", err, ) s.errs.WithLabelValues(u.Host).Inc() @@ -344,7 +395,7 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { s.sent.WithLabelValues(u.Host).Add(float64(len(alerts))) atomic.AddUint64(&numSuccess, 1) - }(am, u) + }(am, *u) } } wg.Wait() @@ -354,7 +405,7 @@ func (s *Sender) Send(ctx context.Context, alerts []*Alert) { } s.dropped.Add(float64(len(alerts))) - level.Warn(s.logger).Log("msg", "failed to send alerts to all alertmanagers", "alerts", string(b)) + level.Warn(s.logger).Log("msg", "failed to send alerts to all alertmanagers", "numAlerts", len(alerts)) } type Dispatcher interface { @@ -369,10 +420,11 @@ type Alertmanager struct { logger log.Logger dispatcher Dispatcher timeout time.Duration + version APIVersion } // NewAlertmanager returns a new Alertmanager client. -func NewAlertmanager(logger log.Logger, dispatcher Dispatcher, timeout time.Duration) *Alertmanager { +func NewAlertmanager(logger log.Logger, dispatcher Dispatcher, timeout time.Duration, version APIVersion) *Alertmanager { if logger == nil { logger = log.NewNopLogger() } @@ -381,11 +433,11 @@ func NewAlertmanager(logger log.Logger, dispatcher Dispatcher, timeout time.Dura logger: logger, dispatcher: dispatcher, timeout: timeout, + version: version, } } func (a *Alertmanager) postAlerts(ctx context.Context, u url.URL, r io.Reader) error { - u.Path = path.Join(u.Path, alertPushEndpoint) req, err := http.NewRequest("POST", u.String(), r) if err != nil { return err diff --git a/pkg/alert/alert_test.go b/pkg/alert/alert_test.go index c3fa5f7cb6..3b99266173 100644 --- a/pkg/alert/alert_test.go +++ b/pkg/alert/alert_test.go @@ -77,7 +77,7 @@ func TestSenderSendsOk(t *testing.T) { poster := &fakeClient{ urls: []*url.URL{{Host: "am1:9090"}, {Host: "am2:9090"}}, } - s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute)}) + s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute, APIv1)}) s.Send(context.Background(), []*Alert{{}, {}}) @@ -104,7 +104,7 @@ func TestSenderSendsOneFails(t *testing.T) { return rec.Result(), nil }, } - s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute)}) + s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute, APIv1)}) s.Send(context.Background(), []*Alert{{}, {}}) @@ -125,7 +125,7 @@ func TestSenderSendsAllFail(t *testing.T) { return nil, errors.New("no such host") }, } - s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute)}) + s := NewSender(nil, nil, []*Alertmanager{NewAlertmanager(nil, poster, time.Minute, APIv1)}) s.Send(context.Background(), []*Alert{{}, {}}) diff --git a/pkg/alert/config.go b/pkg/alert/config.go index 3df4216f5c..e8398b4abb 100644 --- a/pkg/alert/config.go +++ b/pkg/alert/config.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/pkg/errors" "github.com/prometheus/common/model" "gopkg.in/yaml.v2" @@ -19,11 +20,39 @@ type AlertingConfig struct { } // AlertmanagerConfig represents a client to a cluster of Alertmanager endpoints. -// TODO(simonpasquier): add support for API version (v1 or v2). type AlertmanagerConfig struct { HTTPClientConfig http_util.ClientConfig `yaml:"http_config"` EndpointsConfig http_util.EndpointsConfig `yaml:",inline"` Timeout model.Duration `yaml:"timeout"` + APIVersion APIVersion `yaml:"api_version"` +} + +// APIVersion represents the API version of the Alertmanager endpoint. +type APIVersion string + +const ( + APIv1 APIVersion = "v1" + APIv2 APIVersion = "v2" +) + +var supportedAPIVersions = []APIVersion{ + APIv1, APIv2, +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (v *APIVersion) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return errors.Wrap(err, "invalid Alertmanager API version") + } + + for _, ver := range supportedAPIVersions { + if APIVersion(s) == ver { + *v = ver + return nil + } + } + return errors.Errorf("expected Alertmanager API version to be one of %v but got %q", supportedAPIVersions, s) } func DefaultAlertmanagerConfig() AlertmanagerConfig { @@ -33,7 +62,8 @@ func DefaultAlertmanagerConfig() AlertmanagerConfig { StaticAddresses: []string{}, FileSDConfigs: []http_util.FileSDConfig{}, }, - Timeout: model.Duration(time.Second * 10), + Timeout: model.Duration(time.Second * 10), + APIVersion: APIv1, } } diff --git a/pkg/alert/config_test.go b/pkg/alert/config_test.go index 0a50d4da36..9dd7ddab19 100644 --- a/pkg/alert/config_test.go +++ b/pkg/alert/config_test.go @@ -4,10 +4,43 @@ import ( "testing" "time" + "gopkg.in/yaml.v2" + "github.com/thanos-io/thanos/pkg/http" "github.com/thanos-io/thanos/pkg/testutil" ) +func TestUnmarshalAPIVersion(t *testing.T) { + for _, tc := range []struct { + v string + + err bool + expected APIVersion + }{ + { + v: "v1", + expected: APIv1, + }, + { + v: "v3", + err: true, + }, + { + v: "{}", + err: true, + }, + } { + var got APIVersion + err := yaml.Unmarshal([]byte(tc.v), &got) + if tc.err { + testutil.NotOk(t, err) + continue + } + testutil.Ok(t, err) + testutil.Equals(t, tc.expected, got) + } +} + func TestBuildAlertmanagerConfiguration(t *testing.T) { for _, tc := range []struct { address string diff --git a/pkg/testutil/prometheus.go b/pkg/testutil/prometheus.go index 209f4e5175..505320fcaa 100644 --- a/pkg/testutil/prometheus.go +++ b/pkg/testutil/prometheus.go @@ -28,7 +28,7 @@ import ( const ( defaultPrometheusVersion = "v2.13.0" - defaultAlertmanagerVersion = "v0.15.2" + defaultAlertmanagerVersion = "v0.20.0" defaultMinioVersion = "RELEASE.2018-10-06T00-15-16Z" // Space delimited list of versions. diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 757ff7a6d0..e5409cfa51 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -197,12 +197,12 @@ func (m *mockAlertmanager) ServeHTTP(resp http.ResponseWriter, req *http.Request func TestRuleAlertmanagerHTTPClient(t *testing.T) { a := newLocalAddresser() - // Plain HTTP with a prefix. + // API v1 with plain HTTP and a prefix. handler1 := newMockAlertmanager("/prefix/api/v1/alerts", "") srv1 := httptest.NewServer(handler1) defer srv1.Close() - // HTTPS with authentication. - handler2 := newMockAlertmanager("/api/v1/alerts", "secret") + // API v2 with HTTPS and authentication. + handler2 := newMockAlertmanager("/api/v2/alerts", "secret") srv2 := httptest.NewTLSServer(handler2) defer srv2.Close() @@ -225,7 +225,8 @@ func TestRuleAlertmanagerHTTPClient(t *testing.T) { Scheme: "http", PathPrefix: "/prefix/", }, - Timeout: model.Duration(time.Second), + Timeout: model.Duration(time.Second), + APIVersion: alert.APIv1, }, alert.AlertmanagerConfig{ HTTPClientConfig: http_util.ClientConfig{ @@ -238,7 +239,8 @@ func TestRuleAlertmanagerHTTPClient(t *testing.T) { StaticAddresses: []string{srv2.Listener.Addr().String()}, Scheme: "https", }, - Timeout: model.Duration(time.Second), + Timeout: model.Duration(time.Second), + APIVersion: alert.APIv2, }, ) @@ -310,7 +312,8 @@ func TestRuleAlertmanagerFileSD(t *testing.T) { }, Scheme: "http", }, - Timeout: model.Duration(time.Second), + Timeout: model.Duration(time.Second), + APIVersion: alert.APIv1, }, ) @@ -413,7 +416,8 @@ func TestRule(t *testing.T) { StaticAddresses: []string{am.HTTP.HostPort()}, Scheme: "http", }, - Timeout: model.Duration(time.Second), + Timeout: model.Duration(time.Second), + APIVersion: alert.APIv2, }, ) @@ -681,7 +685,8 @@ func TestRulePartialResponse(t *testing.T) { StaticAddresses: []string{am.HTTP.HostPort()}, Scheme: "http", }, - Timeout: model.Duration(time.Second), + Timeout: model.Duration(time.Second), + APIVersion: alert.APIv2, }, ) From fbe1c750af84dd99b3f9f3c22843a427606fec1e Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Tue, 14 Jan 2020 22:14:04 +0800 Subject: [PATCH 176/257] do not mark bug label as stale (#1991) * do not mark bug label as stale Signed-off-by: Xiang Dai <764524258@qq.com> * feedback Signed-off-by: Xiang Dai <764524258@qq.com> --- .github/stale.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/stale.yml b/.github/stale.yml index 4bd5519c11..0b024c09ed 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -13,6 +13,8 @@ onlyLabels: [] # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable exemptLabels: - "priority: P0" + - bug + - pinned # Set to true to ignore issues in a project (defaults to false) exemptProjects: false From 211a50a31f8894da7c1693d2120cc4a8cf9597bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Tue, 14 Jan 2020 21:40:52 +0200 Subject: [PATCH 177/257] docs/release-process: add more steps (#1997) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add one more step for metrics and a note that we should tag the merge commit due to the reasons mentioned. Signed-off-by: Giedrius Statkevičius --- docs/release-process.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/release-process.md b/docs/release-process.md index fdf488b88b..a535c85de4 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -66,6 +66,10 @@ Release is happening on separate `release-.` branch. 1. *In case of version after `v1+.y.z`*, double check if none of the changes break API compatibility. This should be done in PR review process, but double check is good to have. 1. In case of `v0.y.z`, document all incompatibilities in changelog. +1. Double check metric changes: + 1. Note any changes in the changelog + 1. If there were any changes then update the relevant alerting rules and/or dashboards since `thanos-mixin` is part of the repository now + 1. Update tutorials: 1. Update the Thanos version used in the [tutorials](../tutorials) manifests. 1. In case of any breaking changes or necessary updates adjust the manifests @@ -82,6 +86,8 @@ Release is happening on separate `release-.` branch. Signing a tag with a GPG key is appreciated, but in case you can't add a GPG key to your Github account using the following [procedure](https://help.github.com/articles/generating-a-gpg-key/), you can replace the `-s` flag by `-a` flag of the `git tag` command to only annotate the tag without signing. + Please make sure that you are tagging the merge commit because otherwise GitHub's UI will show that there were more commits after your release. + 1. Once a tag is created, the release process through CircleCI will be triggered for this tag. 1. You must create a Github Release using the UI for this tag, as otherwise CircleCI will not be able to upload tarballs for this tag. Also, you must create the Github Release using a Github user that has granted access rights to CircleCI. List of maintainers is available [here](/MAINTAINERS.md) From 9545b12a29b942c4a688eac7648b46e8f53a6976 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Tue, 14 Jan 2020 21:09:22 +0100 Subject: [PATCH 178/257] CHANGELOG.md: fix v0.10.0 entry (#1996) Some pull requests were incorrectly added to v0.10.0 while they're not released yet. Signed-off-by: Simon Pasquier --- CHANGELOG.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7e187c983..e8feac8514 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,18 +11,26 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased +### Fixed + +- [#1985](https://github.com/thanos-io/thanos/pull/1985) store gateway: Fixed case where series entry is larger than 64KB in index. + ### Added - [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags - [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction - [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`. +- [#1939](https://github.com/thanos-io/thanos/pull/1939) Ruler: Add TLS and authentication support for query endpoints with the `--query.config` and `--query.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. - [#1982](https://github.com/thanos-io/thanos/pull/1982) Ruler: Add support for Alertmanager v2 API endpoints. +### Changed + +- [#1933](https://github.com/thanos-io/thanos/pull/1933) Add a flag `--tsdb.wal-compression` to configure whether to enable tsdb wal compression in ruler and receiver. + ## [v0.10.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0) - 2020.01.13 ### Fixed -- [1985](https://github.com/thanos-io/thanos/pull/1985) store gateway: Fixed case where series entry is larger than 64KB in index. - [#1919](https://github.com/thanos-io/thanos/issues/1919) Compactor: Fixed potential data loss when uploading older blocks, or upload taking long time while compactor is running. - [#1937](https://github.com/thanos-io/thanos/pull/1937) Compactor: Improved synchronization of meta JSON files. @@ -50,7 +58,6 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1881](https://github.com/thanos-io/thanos/pull/1881) Store Gateway: memcached support for index cache. See [documentation](docs/components/store.md/#index-cache) for further information. - [#1904](https://github.com/thanos-io/thanos/pull/1904) Add a skip-chunks option in Store Series API to improve the response time of `/api/v1/series` endpoint. - [#1910](https://github.com/thanos-io/thanos/pull/1910) Query: `/api/v1/labels` now understands `POST` - useful for sending bigger requests -- [#1939](https://github.com/thanos-io/thanos/pull/1939) Ruler: Add TLS and authentication support for query endpoints with the `--query.config` and `--query.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. ### Changed @@ -63,7 +70,6 @@ Compactor now properly handles partial block uploads for all operation like rete - [#1833](https://github.com/thanos-io/thanos/pull/1833) `--shipper.upload-compacted` flag has been promoted to non hidden, non experimental state. More info available [here](docs/quick-tutorial.md#uploading-old-metrics). - [#1867](https://github.com/thanos-io/thanos/pull/1867) Ruler: now sets a `Thanos/$version` `User-Agent` in requests - [#1887](https://github.com/thanos-io/thanos/pull/1887) Service discovery now deduplicates targets between different target groups -- [#1933](https://github.com/thanos-io/thanos/pull/1933) Add a flag `--tsdb.wal-compression` to configure whether to enable tsdb wal compression in ruler and receiver. ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03 From 46a97fdc9765bbe7eb9d75d801abf186015e1b01 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Wed, 15 Jan 2020 02:50:10 +0100 Subject: [PATCH 179/257] mixin: Add Thanos Ruler alerts (#1963) * Add Thanos Ruler alerts Signed-off-by: Kemal Akkoyun * Fix wrong job selector Signed-off-by: Kemal Akkoyun * Add pod label as aggregator for evaluation latency query Signed-off-by: Kemal Akkoyun * Fix review issues Signed-off-by: Kemal Akkoyun --- examples/alerts/alerts.md | 91 ++++++++--- examples/alerts/alerts.yaml | 77 +++++++++ examples/dashboards/ruler.json | 198 ++++++++++++++++++++++-- mixin/thanos/alerts/alerts.libsonnet | 1 + mixin/thanos/alerts/ruler.libsonnet | 121 +++++++++++++++ mixin/thanos/dashboards/ruler.libsonnet | 17 ++ 6 files changed, 467 insertions(+), 38 deletions(-) create mode 100644 mixin/thanos/alerts/ruler.libsonnet diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index 14367defa7..41e135b24c 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -64,39 +64,84 @@ rules: For Thanos ruler we run some alerts in local Prometheus, to make sure that Thanos Rule is working: -[//]: # "TODO(kakkoyun): Generate rule rules using thanos-mixin." - +[embedmd]:# (../tmp/thanos-ruler.rules.yaml yaml) ```yaml -- alert: ThanosRuleIsDown - expr: up{app="thanos-ruler"} == 0 or absent(up{app="thanos-ruler"}) +name: thanos-ruler.rules +rules: +- alert: ThanosRulerQueueIsDroppingAlerts + annotations: + message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to queue alerts. + expr: | + sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=~"thanos-ruler.*"}[5m])) > 0 for: 5m labels: - team: TEAM + severity: critical +- alert: ThanosRulerSenderIsFailingAlerts annotations: - summary: Thanos Rule is down - impact: Alerts are not working - action: 'check {{ $labels.kubernetes_pod_name }} pod in {{ $labels.kubernetes_namespace}} namespace' - dashboard: RULE_DASHBOARD -- alert: ThanosRuleIsDroppingAlerts - expr: rate(thanos_alert_queue_alerts_dropped_total{app="thanos-ruler"}[5m]) > 0 + message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to send alerts + to alertmanager. + expr: | + sum by (job) (rate(thanos_alert_sender_alerts_dropped_total{job=~"thanos-ruler.*"}[5m])) > 0 for: 5m labels: - team: TEAM + severity: critical +- alert: ThanosRulerHighRuleExaluationFailures annotations: - summary: Thanos Rule is dropping alerts - impact: Alerts are not working - action: 'check {{ $labels.kubernetes_pod_name }} pod logs in {{ $labels.kubernetes_namespace}} namespace' - dashboard: RULE_DASHBOARD -- alert: ThanosRuleGrpcErrorRate - expr: rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable",app="thanos-ruler"}[5m]) > 0 + message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to evaluate rules. + expr: | + ( + sum by (job) (rate(prometheus_rule_evaluation_failures_total{job=~"thanos-ruler.*"}[5m])) + / + sum by (job) (rate(prometheus_rule_evaluations_total{job=~"thanos-ruler.*"}[5m])) + * 100 > 5 + ) for: 5m labels: - team: TEAM + severity: warning +- alert: ThanosRulerHighRuleExaluationWarnings annotations: - summary: Thanos Rule is returning Internal/Unavailable errors - impact: Recording Rules are not working - action: Check {{ $labels.kubernetes_pod_name }} pod logs in {{ $labels.kubernetes_namespace}} namespace - dashboard: RULE_DASHBOARD + message: Thanos Ruler {{$labels.job}} {{$labels.pod}} has high number of evaluation + warnings. + expr: | + sum by (job) (rate(thanos_rule_evaluation_with_warnings_total{job=~"thanos-ruler.*"}[5m])) > 0 + for: 15m + labels: + severity: warning +- alert: ThanosRulerRuleEvaluationLatencyHigh + annotations: + message: Thanos Ruler {{$labels.job}}/{{$labels.pod}} has higher evaluation latency + than interval for {{$labels.rule_group}}. + expr: | + ( + sum by (job, pod, rule_group) (prometheus_rule_group_last_duration_seconds{job=~"thanos-ruler.*"}) + > + sum by (job, pod, rule_group) (prometheus_rule_group_interval_seconds{job=~"thanos-ruler.*"}) + ) + for: 5m + labels: + severity: warning +- alert: ThanosRulerGrpcErrorRate + annotations: + message: Thanos Ruler {{$labels.job}} is failing to handle {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-ruler.*"}[5m])) + / + sum by (job) (rate(grpc_server_started_total{job=~"thanos-ruler.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: warning +- alert: ThanosRulerConfigReloadFailure + annotations: + message: Thanos Ruler {{$labels.job}} has not been able to reload its configuration. + expr: avg(thanos_rule_config_last_reload_successful{job=~"thanos-ruler.*"}) by (job) + != 1 + for: 5m + labels: + severity: warning ``` ## Store Gateway diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index 23f10b655f..0eb4fda1d1 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -277,6 +277,83 @@ groups: for: 10m labels: severity: warning +- name: thanos-ruler.rules + rules: + - alert: ThanosRulerQueueIsDroppingAlerts + annotations: + message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to queue alerts. + expr: | + sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=~"thanos-ruler.*"}[5m])) > 0 + for: 5m + labels: + severity: critical + - alert: ThanosRulerSenderIsFailingAlerts + annotations: + message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to send alerts + to alertmanager. + expr: | + sum by (job) (rate(thanos_alert_sender_alerts_dropped_total{job=~"thanos-ruler.*"}[5m])) > 0 + for: 5m + labels: + severity: critical + - alert: ThanosRulerHighRuleExaluationFailures + annotations: + message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to evaluate + rules. + expr: | + ( + sum by (job) (rate(prometheus_rule_evaluation_failures_total{job=~"thanos-ruler.*"}[5m])) + / + sum by (job) (rate(prometheus_rule_evaluations_total{job=~"thanos-ruler.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: warning + - alert: ThanosRulerHighRuleExaluationWarnings + annotations: + message: Thanos Ruler {{$labels.job}} {{$labels.pod}} has high number of evaluation + warnings. + expr: | + sum by (job) (rate(thanos_rule_evaluation_with_warnings_total{job=~"thanos-ruler.*"}[5m])) > 0 + for: 15m + labels: + severity: warning + - alert: ThanosRulerRuleEvaluationLatencyHigh + annotations: + message: Thanos Ruler {{$labels.job}}/{{$labels.pod}} has higher evaluation + latency than interval for {{$labels.rule_group}}. + expr: | + ( + sum by (job, pod, rule_group) (prometheus_rule_group_last_duration_seconds{job=~"thanos-ruler.*"}) + > + sum by (job, pod, rule_group) (prometheus_rule_group_interval_seconds{job=~"thanos-ruler.*"}) + ) + for: 5m + labels: + severity: warning + - alert: ThanosRulerGrpcErrorRate + annotations: + message: Thanos Ruler {{$labels.job}} is failing to handle {{ $value | humanize + }}% of requests. + expr: | + ( + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-ruler.*"}[5m])) + / + sum by (job) (rate(grpc_server_started_total{job=~"thanos-ruler.*"}[5m])) + * 100 > 5 + ) + for: 5m + labels: + severity: warning + - alert: ThanosRulerConfigReloadFailure + annotations: + message: Thanos Ruler {{$labels.job}} has not been able to reload its configuration. + expr: avg(thanos_rule_config_last_reload_successful{job=~"thanos-ruler.*"}) by + (job) != 1 + for: 5m + labels: + severity: warning - name: thanos-component-absent.rules rules: - alert: ThanosCompactorIsDown diff --git a/examples/dashboards/ruler.json b/examples/dashboards/ruler.json index fa2596da4a..6dc5f85a25 100644 --- a/examples/dashboards/ruler.json +++ b/examples/dashboards/ruler.json @@ -347,6 +347,174 @@ "title": "Alert Sent", "titleSize": "h6" }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows rate of queued alerts.", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_alert_queue_alerts_dropped_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Push Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "error": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows ratio of dropped alerts compared to the total number of queued alerts.", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(thanos_alert_queue_alerts_dropped_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) / sum(rate(thanos_alert_queue_alerts_pushed_total{namespace=\"$namespace\",job=~\"$job\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "error", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Drop Ratio", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alert Queue", + "titleSize": "h6" + }, { "collapse": false, "height": "250px", @@ -378,7 +546,7 @@ "datasource": "$datasource", "description": "Shows rate of handled Unary gRPC requests.", "fill": 10, - "id": 5, + "id": 7, "legend": { "avg": false, "current": false, @@ -457,7 +625,7 @@ "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests.", "fill": 10, - "id": 6, + "id": 8, "legend": { "avg": false, "current": false, @@ -534,7 +702,7 @@ "datasource": "$datasource", "description": "Shows how long has it taken to handle requests, in quantiles.", "fill": 1, - "id": 7, + "id": 9, "legend": { "avg": false, "current": false, @@ -639,7 +807,7 @@ "datasource": "$datasource", "description": "Shows rate of handled Unary gRPC requests.", "fill": 10, - "id": 8, + "id": 10, "legend": { "avg": false, "current": false, @@ -716,7 +884,7 @@ "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests.", "fill": 10, - "id": 9, + "id": 11, "legend": { "avg": false, "current": false, @@ -793,7 +961,7 @@ "datasource": "$datasource", "description": "Shows how long has it taken to handle requests, in quantiles.", "fill": 1, - "id": 10, + "id": 12, "legend": { "avg": false, "current": false, @@ -917,7 +1085,7 @@ "datasource": "$datasource", "description": "Shows rate of handled Streamed gRPC requests.", "fill": 10, - "id": 11, + "id": 13, "legend": { "avg": false, "current": false, @@ -996,7 +1164,7 @@ "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests.", "fill": 10, - "id": 12, + "id": 14, "legend": { "avg": false, "current": false, @@ -1073,7 +1241,7 @@ "datasource": "$datasource", "description": "Shows how long has it taken to handle requests, in quantiles", "fill": 1, - "id": 13, + "id": 15, "legend": { "avg": false, "current": false, @@ -1178,7 +1346,7 @@ "datasource": "$datasource", "description": "Shows rate of handled Streamed gRPC requests.", "fill": 10, - "id": 14, + "id": 16, "legend": { "avg": false, "current": false, @@ -1255,7 +1423,7 @@ "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests.", "fill": 10, - "id": 15, + "id": 17, "legend": { "avg": false, "current": false, @@ -1332,7 +1500,7 @@ "datasource": "$datasource", "description": "Shows how long has it taken to handle requests, in quantiles", "fill": 1, - "id": 16, + "id": 18, "legend": { "avg": false, "current": false, @@ -1436,7 +1604,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 17, + "id": 19, "legend": { "avg": false, "current": false, @@ -1552,7 +1720,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 18, + "id": 20, "legend": { "avg": false, "current": false, @@ -1628,7 +1796,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 19, + "id": 21, "legend": { "avg": false, "current": false, diff --git a/mixin/thanos/alerts/alerts.libsonnet b/mixin/thanos/alerts/alerts.libsonnet index 0eb63dc98d..e3fa004090 100644 --- a/mixin/thanos/alerts/alerts.libsonnet +++ b/mixin/thanos/alerts/alerts.libsonnet @@ -3,4 +3,5 @@ (import 'receiver.libsonnet') + (import 'sidecar.libsonnet') + (import 'store.libsonnet') + +(import 'ruler.libsonnet') + (import 'absent.libsonnet') diff --git a/mixin/thanos/alerts/ruler.libsonnet b/mixin/thanos/alerts/ruler.libsonnet new file mode 100644 index 0000000000..08de10a23b --- /dev/null +++ b/mixin/thanos/alerts/ruler.libsonnet @@ -0,0 +1,121 @@ +{ + local thanos = self, + ruler+:: { + jobPrefix: error 'must provide job prefix for Thanos Ruler alerts', + selector: error 'must provide selector for Thanos Ruler alerts', + }, + prometheusAlerts+:: { + groups+: [ + { + name: 'thanos-ruler.rules', + rules: [ + { + alert: 'ThanosRulerQueueIsDroppingAlerts', + annotations: { + message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to queue alerts.', + }, + expr: ||| + sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{%(selector)s}[5m])) > 0 + ||| % thanos.ruler, + 'for': '5m', + labels: { + severity: 'critical', + }, + }, + { + alert: 'ThanosRulerSenderIsFailingAlerts', + annotations: { + message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to send alerts to alertmanager.', + }, + expr: ||| + sum by (job) (rate(thanos_alert_sender_alerts_dropped_total{%(selector)s}[5m])) > 0 + ||| % thanos.ruler, + 'for': '5m', + labels: { + severity: 'critical', + }, + }, + { + alert: 'ThanosRulerHighRuleExaluationFailures', + annotations: { + message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to evaluate rules.', + }, + expr: ||| + ( + sum by (job) (rate(prometheus_rule_evaluation_failures_total{%(selector)s}[5m])) + / + sum by (job) (rate(prometheus_rule_evaluations_total{%(selector)s}[5m])) + * 100 > 5 + ) + ||| % thanos.ruler, + + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosRulerHighRuleExaluationWarnings', + annotations: { + message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} has high number of evaluation warnings.', + }, + expr: ||| + sum by (job) (rate(thanos_rule_evaluation_with_warnings_total{%(selector)s}[5m])) > 0 + ||| % thanos.ruler, + + 'for': '15m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosRulerRuleEvaluationLatencyHigh', + annotations: { + message: 'Thanos Ruler {{$labels.job}}/{{$labels.pod}} has higher evaluation latency than interval for {{$labels.rule_group}}.', + }, + expr: ||| + ( + sum by (job, pod, rule_group) (prometheus_rule_group_last_duration_seconds{%(selector)s}) + > + sum by (job, pod, rule_group) (prometheus_rule_group_interval_seconds{%(selector)s}) + ) + ||| % thanos.ruler, + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosRulerGrpcErrorRate', + annotations: { + message: 'Thanos Ruler {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', + }, + expr: ||| + ( + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", %(selector)s}[5m])) + / + sum by (job) (rate(grpc_server_started_total{%(selector)s}[5m])) + * 100 > 5 + ) + ||| % thanos.ruler, + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosRulerConfigReloadFailure', + annotations: { + message: 'Thanos Ruler {{$labels.job}} has not been able to reload its configuration.', + }, + expr: 'avg(thanos_rule_config_last_reload_successful{%(selector)s}) by (job) != 1' % thanos.ruler, + 'for': '5m', + labels: { + severity: 'warning', + }, + }, + ], + }, + ], + }, +} diff --git a/mixin/thanos/dashboards/ruler.libsonnet b/mixin/thanos/dashboards/ruler.libsonnet index 941c38b417..067e4d1af2 100644 --- a/mixin/thanos/dashboards/ruler.libsonnet +++ b/mixin/thanos/dashboards/ruler.libsonnet @@ -39,6 +39,23 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; g.latencyPanel('thanos_alert_sender_latency_seconds', 'namespace="$namespace",job=~"$job"'), ) ) + .addRow( + g.row('Alert Queue') + .addPanel( + g.panel('Push Rate', 'Shows rate of queued alerts.') + + g.queryPanel( + 'sum(rate(thanos_alert_queue_alerts_dropped_total{namespace="$namespace",job=~"$job"}[$interval])) by (job, pod)', + '{{job}} {{pod}}' + ) + ) + .addPanel( + g.panel('Drop Ratio', 'Shows ratio of dropped alerts compared to the total number of queued alerts.') + + g.qpsErrTotalPanel( + 'thanos_alert_queue_alerts_dropped_total{namespace="$namespace",job=~"$job"}', + 'thanos_alert_queue_alerts_pushed_total{namespace="$namespace",job=~"$job"}', + ) + ) + ) .addRow( g.row('gRPC (Unary)') .addPanel( From 01bfca2895306e9a1df6340b2c950350c278ac6f Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 17 Jan 2020 17:11:58 +0100 Subject: [PATCH 180/257] Fix scripts/verify-vendor.sh Signed-off-by: Simon Pasquier --- scripts/verify-vendor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/verify-vendor.sh b/scripts/verify-vendor.sh index 076c0963af..5a24202a2f 100755 --- a/scripts/verify-vendor.sh +++ b/scripts/verify-vendor.sh @@ -1,6 +1,6 @@ #!/bin/bash -if grep "/vendor" .gitignore; then +if grep "^/vendor" .gitignore; then echo '"/vendor" directory found in .gitignore. Stopping as this breaks further checks' exit 1 fi From 171889da3617543d82cd887030b16f9372474869 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 21 Jan 2020 14:33:37 +0100 Subject: [PATCH 181/257] Added binary index header implementation. (#1952) * Added binary index header implementation with benchmarks. This PR adds index-header implementation based on [this design](https://thanos.io/proposals/201912_thanos_binary_index_header.md/) It adds a separate indexheader.Binary* structs and method allowing to build and read index-header in binary format. Size difference: 10k series for my autogenerated data it's 2.1x -rw-r--r-- 1 bwplotka bwplotka 6.1M Jan 10 13:20 index -rw-r--r-- 1 bwplotka bwplotka 23K Jan 10 13:20 index.cache.json -rw-r--r-- 1 bwplotka bwplotka 9.2K Jan 10 13:20 index-header For realistic block 8mln series, also similar gain. -rw-r--r-- 1 bwplotka bwplotka 1.9G Jan 10 13:29 index -rw-r--r-- 1 bwplotka bwplotka 287M Jan 10 13:29 index.cache.json -rw-r--r-- 1 bwplotka bwplotka 122M Jan 10 13:29 index-header NOTE: Size is smaller, but it's not what we are trying to optimize for. Nevertheless PostingOffsets and Symbols takes significant amount of bytes. The only downsides of size is the fact that to create such index-header we have to fetch those two parts ~60MB each from object storage. Idea for improvement if that will become a problem: Cache only 32th of the posting ranges and fetch gaps between on demand on query time (with some cache). Real time latencies for creation and loading (without network traffic): For 10k block it's similar for both (ms/micros), for 8mln we can spot the difference: index-header: * write 134.197732ms * read 415.971774ms index-cache.json: * write 6.712496338s * read 6.112222132s Before comparing I changed names to correlate tests: BenchmarkJSONReader-12-> BenchmarkRead-12 old BenchmarkBinaryReader-12 -> BenchmarkRead-12 new BenchmarkJSONWrite-12 -> BenchmarkWrite-12 old BenchmarkBinaryWrite-12 -> BenchmarkWrite-12 new benchmark old ns/op new ns/op delta BenchmarkRead-12 591780 66613 -88.74% BenchmarkWrite-12 2458454 6532651 +165.72% benchmark old allocs new allocs delta BenchmarkRead-12 2306 629 -72.72% BenchmarkWrite-12 1995 64 -96.79% benchmark old bytes new bytes delta BenchmarkRead-12 150904 32976 -78.15% BenchmarkWrite-12 161501 73412 -54.54% CPU time for smaller index file is interesting. Value is low anyway. Might be something to follow up. benchmark old ns/op new ns/op delta BenchmarkRead-12 7026290474 552913402 -92.13% BenchmarkWrite-12 6480769814 276441977 -95.73% benchmark old allocs new allocs delta BenchmarkRead-12 20100014 5501312 -72.63% BenchmarkWrite-12 18263356 64 -100.00% benchmark old bytes new bytes delta BenchmarkRead-12 1873789526 406021516 -78.33% BenchmarkWrite-12 2385193317 74187 -100.00% Signed-off-by: Bartlomiej Plotka * Enabled index-header under experimental flag. (#1986) Enabled it also on all our tests. Depends on: https://github.com/thanos-io/thanos/pull/1952 Signed-off-by: Bartlomiej Plotka * Addressed comments. Signed-off-by: Bartlomiej Plotka * Fixed small bug in resize. Signed-off-by: Bartlomiej Plotka --- cmd/thanos/store.go | 9 + docs/components/store.md | 59 ++ go.mod | 2 +- go.sum | 4 +- pkg/block/block.go | 4 +- pkg/block/block_test.go | 35 +- pkg/block/indexheader/binary_reader.go | 817 ++++++++++++++++++ pkg/block/indexheader/header.go | 30 +- pkg/block/indexheader/header_test.go | 383 +++++++- pkg/block/indexheader/json_reader.go | 49 +- .../testdata/index_format_v1/chunks/.gitkeep | 0 .../testdata/index_format_v1/index | Bin 0 -> 5407 bytes .../testdata/index_format_v1/meta.json | 17 + .../testdata/index_format_v2/chunks/.gitkeep | 0 .../testdata/index_format_v2/index | Bin 0 -> 6372676 bytes .../testdata/index_format_v2/meta.json | 27 + pkg/store/bucket.go | 142 ++- pkg/store/bucket_e2e_test.go | 74 +- pkg/store/bucket_test.go | 35 +- pkg/testutil/copy.go | 49 ++ test/e2e/spinup_test.go | 1 + 21 files changed, 1579 insertions(+), 158 deletions(-) create mode 100644 pkg/block/indexheader/binary_reader.go create mode 100644 pkg/block/indexheader/testdata/index_format_v1/chunks/.gitkeep create mode 100644 pkg/block/indexheader/testdata/index_format_v1/index create mode 100644 pkg/block/indexheader/testdata/index_format_v1/meta.json create mode 100644 pkg/block/indexheader/testdata/index_format_v2/chunks/.gitkeep create mode 100644 pkg/block/indexheader/testdata/index_format_v2/index create mode 100644 pkg/block/indexheader/testdata/index_format_v2/meta.json create mode 100644 pkg/testutil/copy.go diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 6937b9f4df..5e19037be1 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -75,6 +75,9 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { selectorRelabelConf := regSelectorRelabelFlags(cmd) + enableIndexHeader := cmd.Flag("experimental.enable-index-header", "If true, Store Gateway will recreate index-header instead of index-cache.json for each block. This will replace index-cache.json permanently once it will be out of experimental stage."). + Hidden().Default("false").Bool() + m[component.Store.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, debugLogging bool) error { if minTime.PrometheusTimestamp() > maxTime.PrometheusTimestamp() { return errors.Errorf("invalid argument: --min-time '%s' can't be greater than --max-time '%s'", @@ -109,6 +112,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { }, selectorRelabelConf, *advertiseCompatibilityLabel, + *enableIndexHeader, ) } } @@ -140,6 +144,7 @@ func runStore( filterConf *store.FilterConfig, selectorRelabelConf *extflag.PathOrContent, advertiseCompatibilityLabel bool, + enableIndexHeader bool, ) error { // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. statusProber := prober.New(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) @@ -214,6 +219,9 @@ func runStore( return errors.Wrap(err, "meta fetcher") } + if enableIndexHeader { + level.Info(logger).Log("msg", "index-header instead of index-cache.json enabled") + } bs, err := store.NewBucketStore( logger, reg, @@ -228,6 +236,7 @@ func runStore( blockSyncConcurrency, filterConf, advertiseCompatibilityLabel, + enableIndexHeader, ) if err != nil { return errors.Wrap(err, "create object storage store") diff --git a/docs/components/store.md b/docs/components/store.md index f071e9475e..2cc130d9e9 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -221,3 +221,62 @@ While the remaining settings are **optional**: - `max_get_multi_concurrency`: maximum number of concurrent connections when fetching keys. If set to `0`, the concurrency is unlimited. - `max_get_multi_batch_size`: maximum number of keys a single underlying operation should fetch. If more keys are specified, internally keys are splitted into multiple batches and fetched concurrently, honoring `max_get_multi_concurrency`. If set to `0`, the batch size is unlimited. - `dns_provider_update_interval`: the DNS discovery update interval. + + +## Index Header + +In order to query series inside blocks from object storage, Store Gateway has to know certain initial info about each block such as: + +* symbols table to unintern string values +* postings offset for posting lookup + +In order to achieve so, on startup for each block `index-header` is built from pieces of original block's index and stored on disk. +Such `index-header` file is then mmaped and used by Store Gateway. + +### Format (version 1) + +The following describes the format of the `index-header` file found in each block store gateway local directory. +It is terminated by a table of contents which serves as an entry point into the index. + +``` +┌─────────────────────────────┬───────────────────────────────┐ +│ magic(0xBAAAD792) <4b> │ version(1) <1 byte> │ +├─────────────────────────────┬───────────────────────────────┤ +│ index version(2) <1 byte> │ index PostingOffsetTable <8b> │ +├─────────────────────────────┴───────────────────────────────┤ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ Symbol Table (exact copy from original index) │ │ +│ ├─────────────────────────────────────────────────────────┤ │ +│ │ Posting Offset Table (exact copy from index) │ │ +│ ├─────────────────────────────────────────────────────────┤ │ +│ │ TOC │ │ +│ └─────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +When the index is written, an arbitrary number of padding bytes may be added between the lined out main sections above. When sequentially scanning through the file, any zero bytes after a section's specified length must be skipped. + +Most of the sections described below start with a `len` field. It always specifies the number of bytes just before the trailing CRC32 checksum. The checksum is always calculated over those `len` bytes. + +### Symbol Table + +See [Symbols](https://github.com/prometheus/prometheus/blob/d782387f814753b0118d402ec8cdbdef01bf9079/tsdb/docs/format/index.md#symbol-table) + +### Postings Offset Table + +See [Posting Offset Table](https://github.com/prometheus/prometheus/blob/d782387f814753b0118d402ec8cdbdef01bf9079/tsdb/docs/format/index.md#postings-offset-table) + +### TOC + +The table of contents serves as an entry point to the entire index and points to various sections in the file. +If a reference is zero, it indicates the respective section does not exist and empty results should be returned upon lookup. + +``` +┌─────────────────────────────────────────┐ +│ ref(symbols) <8b> │ +├─────────────────────────────────────────┤ +│ ref(postings offset table) <8b> │ +├─────────────────────────────────────────┤ +│ CRC32 <4b> │ +└─────────────────────────────────────────┘ +``` diff --git a/go.mod b/go.mod index 4f9bf1c077..f401230235 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 github.com/prometheus/common v0.7.0 github.com/prometheus/procfs v0.0.6 // indirect - github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef // master ~ v2.15.2 + github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 // master ~ v2.15.2 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/smartystreets/assertions v1.0.1 // indirect diff --git a/go.sum b/go.sum index d122df9d2c..a101015a5a 100644 --- a/go.sum +++ b/go.sum @@ -501,8 +501,8 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.6 h1:0qbH+Yqu/cj1ViVLvEWCP6qMQ4efWUj6bQqOEA0V0U4= github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= -github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef h1:pYYKXo/zGx25kyViw+Gdbxd0ItIg+vkVKpwgWUEyIc4= -github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI= +github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 h1:uEq+8hKI4kfycPLSKNw844YYkdMNpC2eZpov73AvlFk= +github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= diff --git a/pkg/block/block.go b/pkg/block/block.go index 6fb43505cc..147a82f068 100644 --- a/pkg/block/block.go +++ b/pkg/block/block.go @@ -27,8 +27,10 @@ const ( MetaFilename = "meta.json" // IndexFilename is the known index file for block index. IndexFilename = "index" - // IndexCacheFilename is the canonical name for index cache file that stores essential information needed. + // IndexCacheFilename is the canonical name for json index cache file that stores essential information. IndexCacheFilename = "index.cache.json" + // IndexHeaderFilename is the canonical name for binary index header file that stores essential information. + IndexHeaderFilename = "index-header" // ChunksDirname is the known dir name for chunks with compressed samples. ChunksDirname = "chunks" diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index 18f71c982e..35d22871f5 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -2,7 +2,6 @@ package block import ( "context" - "io" "io/ioutil" "os" "path" @@ -12,7 +11,6 @@ import ( "github.com/fortytw2/leaktest" "github.com/go-kit/kit/log" - "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/testutil" @@ -104,7 +102,7 @@ func TestUpload(t *testing.T) { testutil.NotOk(t, err) testutil.Assert(t, strings.HasSuffix(err.Error(), "/meta.json: no such file or directory"), "") } - testutil.Ok(t, cpy(path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename))) + testutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename)) { // Missing chunks. err := Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String())) @@ -115,7 +113,7 @@ func TestUpload(t *testing.T) { testutil.Equals(t, 1, len(bkt.Objects())) } testutil.Ok(t, os.MkdirAll(path.Join(tmpDir, "test", b1.String(), ChunksDirname), os.ModePerm)) - testutil.Ok(t, cpy(path.Join(tmpDir, b1.String(), ChunksDirname, "000001"), path.Join(tmpDir, "test", b1.String(), ChunksDirname, "000001"))) + testutil.Copy(t, path.Join(tmpDir, b1.String(), ChunksDirname, "000001"), path.Join(tmpDir, "test", b1.String(), ChunksDirname, "000001")) { // Missing index file. err := Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String())) @@ -125,7 +123,7 @@ func TestUpload(t *testing.T) { // Only debug meta.json present. testutil.Equals(t, 1, len(bkt.Objects())) } - testutil.Ok(t, cpy(path.Join(tmpDir, b1.String(), IndexFilename), path.Join(tmpDir, "test", b1.String(), IndexFilename))) + testutil.Copy(t, path.Join(tmpDir, b1.String(), IndexFilename), path.Join(tmpDir, "test", b1.String(), IndexFilename)) testutil.Ok(t, os.Remove(path.Join(tmpDir, "test", b1.String(), MetaFilename))) { // Missing meta.json file. @@ -136,7 +134,7 @@ func TestUpload(t *testing.T) { // Only debug meta.json present. testutil.Equals(t, 1, len(bkt.Objects())) } - testutil.Ok(t, cpy(path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename))) + testutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename)) { // Full block. testutil.Ok(t, Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()))) @@ -170,31 +168,6 @@ func TestUpload(t *testing.T) { } } -func cpy(src, dst string) error { - sourceFileStat, err := os.Stat(src) - if err != nil { - return err - } - - if !sourceFileStat.Mode().IsRegular() { - return errors.Errorf("%s is not a regular file", src) - } - - source, err := os.Open(src) - if err != nil { - return err - } - defer source.Close() - - destination, err := os.Create(dst) - if err != nil { - return err - } - defer destination.Close() - _, err = io.Copy(destination, source) - return err -} - func TestDelete(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() diff --git a/pkg/block/indexheader/binary_reader.go b/pkg/block/indexheader/binary_reader.go new file mode 100644 index 0000000000..095432a80a --- /dev/null +++ b/pkg/block/indexheader/binary_reader.go @@ -0,0 +1,817 @@ +package indexheader + +import ( + "bufio" + "context" + "encoding/binary" + "hash" + "hash/crc32" + "io" + "io/ioutil" + "math" + "os" + "path/filepath" + "sort" + "time" + "unsafe" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/prometheus/tsdb/encoding" + "github.com/prometheus/prometheus/tsdb/fileutil" + "github.com/prometheus/prometheus/tsdb/index" + "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/runutil" +) + +const ( + // BinaryFormatV1 represents first version of index-header file. + BinaryFormatV1 = 1 + + indexTOCLen = 6*8 + crc32.Size + binaryTOCLen = 2*8 + crc32.Size + // headerLen represents number of bytes reserved of index header for header. + headerLen = 4 + 1 + 1 + 8 + + // MagicIndex are 4 bytes at the head of an index-header file. + MagicIndex = 0xBAAAD792 + + symbolFactor = 32 + + postingLengthFieldSize = 4 +) + +// The table gets initialized with sync.Once but may still cause a race +// with any other use of the crc32 package anywhere. Thus we initialize it +// before. +var castagnoliTable *crc32.Table + +func init() { + castagnoliTable = crc32.MakeTable(crc32.Castagnoli) +} + +// newCRC32 initializes a CRC32 hash with a preconfigured polynomial, so the +// polynomial may be easily changed in one location at a later time, if necessary. +func newCRC32() hash.Hash32 { + return crc32.New(castagnoliTable) +} + +// BinaryTOC is a table of content for index-header file. +type BinaryTOC struct { + // Symbols holds start to the same symbols section as index related to this index header. + Symbols uint64 + // PostingsOffsetTable holds start to the the same Postings Offset Table section as index related to this index header. + PostingsOffsetTable uint64 +} + +// WriteBinary build index-header file from the pieces of index in object storage. +func WriteBinary(ctx context.Context, bkt objstore.BucketReader, id ulid.ULID, fn string) (err error) { + ir, indexVersion, err := newChunkedIndexReader(ctx, bkt, id) + if err != nil { + return errors.Wrap(err, "new index reader") + } + + // Buffer for copying and encbuffers. + // This also will control the size of file writer buffer. + buf := make([]byte, 32*1024) + bw, err := newBinaryWriter(fn, buf) + if err != nil { + return errors.Wrap(err, "new binary index header writer") + } + defer runutil.CloseWithErrCapture(&err, bw, "close binary writer for %s", fn) + + if err := bw.AddIndexMeta(indexVersion, ir.toc.PostingsTable); err != nil { + return errors.Wrap(err, "add index meta") + } + + if err := ir.CopySymbols(bw.SymbolsWriter(), buf); err != nil { + return err + } + + if err := bw.f.Flush(); err != nil { + return errors.Wrap(err, "flush") + } + + if err := ir.CopyPostingsOffsets(bw.PostingOffsetsWriter(), buf); err != nil { + return err + } + + if err := bw.f.Flush(); err != nil { + return errors.Wrap(err, "flush") + } + + if err := bw.WriteTOC(); err != nil { + return errors.Wrap(err, "write index header TOC") + } + return nil +} + +type chunkedIndexReader struct { + ctx context.Context + path string + size uint64 + bkt objstore.BucketReader + toc *index.TOC +} + +func newChunkedIndexReader(ctx context.Context, bkt objstore.BucketReader, id ulid.ULID) (*chunkedIndexReader, int, error) { + indexFilepath := filepath.Join(id.String(), block.IndexFilename) + size, err := bkt.ObjectSize(ctx, indexFilepath) + if err != nil { + return nil, 0, errors.Wrapf(err, "get object size of %s", indexFilepath) + } + + rc, err := bkt.GetRange(ctx, indexFilepath, 0, index.HeaderLen) + if err != nil { + return nil, 0, errors.Wrapf(err, "get TOC from object storage of %s", indexFilepath) + } + + b, err := ioutil.ReadAll(rc) + if err != nil { + runutil.CloseWithErrCapture(&err, rc, "close reader") + return nil, 0, errors.Wrapf(err, "get header from object storage of %s", indexFilepath) + } + + if err := rc.Close(); err != nil { + return nil, 0, errors.Wrap(err, "close reader") + } + + if m := binary.BigEndian.Uint32(b[0:4]); m != index.MagicIndex { + return nil, 0, errors.Errorf("invalid magic number %x for %s", m, indexFilepath) + } + + version := int(b[4:5][0]) + + if version != index.FormatV1 && version != index.FormatV2 { + return nil, 0, errors.Errorf("not supported index file version %d of %s", version, indexFilepath) + } + + ir := &chunkedIndexReader{ + ctx: ctx, + path: indexFilepath, + size: size, + bkt: bkt, + } + + toc, err := ir.readTOC() + if err != nil { + return nil, 0, err + } + ir.toc = toc + + return ir, version, nil +} + +func (r *chunkedIndexReader) readTOC() (*index.TOC, error) { + rc, err := r.bkt.GetRange(r.ctx, r.path, int64(r.size-indexTOCLen-crc32.Size), indexTOCLen+crc32.Size) + if err != nil { + return nil, errors.Wrapf(err, "get TOC from object storage of %s", r.path) + } + + tocBytes, err := ioutil.ReadAll(rc) + if err != nil { + runutil.CloseWithErrCapture(&err, rc, "close toc reader") + return nil, errors.Wrapf(err, "get TOC from object storage of %s", r.path) + } + + if err := rc.Close(); err != nil { + return nil, errors.Wrap(err, "close toc reader") + } + + toc, err := index.NewTOCFromByteSlice(realByteSlice(tocBytes)) + if err != nil { + return nil, errors.Wrap(err, "new TOC") + } + return toc, nil +} + +func (r *chunkedIndexReader) CopySymbols(w io.Writer, buf []byte) (err error) { + rc, err := r.bkt.GetRange(r.ctx, r.path, int64(r.toc.Symbols), int64(r.toc.Series-r.toc.Symbols)) + if err != nil { + return errors.Wrapf(err, "get symbols from object storage of %s", r.path) + } + defer runutil.CloseWithErrCapture(&err, rc, "close symbol reader") + + if _, err := io.CopyBuffer(w, rc, buf); err != nil { + return errors.Wrap(err, "copy symbols") + } + + return nil +} + +func (r *chunkedIndexReader) CopyPostingsOffsets(w io.Writer, buf []byte) (err error) { + rc, err := r.bkt.GetRange(r.ctx, r.path, int64(r.toc.PostingsTable), int64(r.size-r.toc.PostingsTable)) + if err != nil { + return errors.Wrapf(err, "get posting offset table from object storage of %s", r.path) + } + defer runutil.CloseWithErrCapture(&err, rc, "close posting offsets reader") + + if _, err := io.CopyBuffer(w, rc, buf); err != nil { + return errors.Wrap(err, "copy posting offsets") + } + + return nil +} + +// TODO(bwplotka): Add padding for efficient read. +type binaryWriter struct { + f *FileWriter + + toc BinaryTOC + + // Reusable memory. + buf encoding.Encbuf + + crc32 hash.Hash +} + +func newBinaryWriter(fn string, buf []byte) (w *binaryWriter, err error) { + dir := filepath.Dir(fn) + + df, err := fileutil.OpenDir(dir) + if err != nil { + return nil, err + } + defer runutil.CloseWithErrCapture(&err, df, "dir close") + + if err := os.RemoveAll(fn); err != nil { + return nil, errors.Wrap(err, "remove any existing index at path") + } + + // We use file writer for buffers not larger than reused one. + f, err := NewFileWriter(fn, len(buf)) + if err != nil { + return nil, err + } + if err := df.Sync(); err != nil { + return nil, errors.Wrap(err, "sync dir") + } + + w = &binaryWriter{ + f: f, + + // Reusable memory. + buf: encoding.Encbuf{B: buf}, + crc32: newCRC32(), + } + + w.buf.Reset() + w.buf.PutBE32(MagicIndex) + w.buf.PutByte(BinaryFormatV1) + + return w, w.f.Write(w.buf.Get()) +} + +type FileWriter struct { + f *os.File + fbuf *bufio.Writer + pos uint64 + name string +} + +// TODO(bwplotka): Added size to method, upstream this. +func NewFileWriter(name string, size int) (*FileWriter, error) { + f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0666) + if err != nil { + return nil, err + } + return &FileWriter{ + f: f, + fbuf: bufio.NewWriterSize(f, size), + pos: 0, + name: name, + }, nil +} + +func (fw *FileWriter) Pos() uint64 { + return fw.pos +} + +func (fw *FileWriter) Write(bufs ...[]byte) error { + for _, b := range bufs { + n, err := fw.fbuf.Write(b) + fw.pos += uint64(n) + if err != nil { + return err + } + // For now the index file must not grow beyond 64GiB. Some of the fixed-sized + // offset references in v1 are only 4 bytes large. + // Once we move to compressed/varint representations in those areas, this limitation + // can be lifted. + if fw.pos > 16*math.MaxUint32 { + return errors.Errorf("%q exceeding max size of 64GiB", fw.name) + } + } + return nil +} + +func (fw *FileWriter) Flush() error { + return fw.fbuf.Flush() +} + +func (fw *FileWriter) WriteAt(buf []byte, pos uint64) error { + if err := fw.Flush(); err != nil { + return err + } + _, err := fw.f.WriteAt(buf, int64(pos)) + return err +} + +// AddPadding adds zero byte padding until the file size is a multiple size. +func (fw *FileWriter) AddPadding(size int) error { + p := fw.pos % uint64(size) + if p == 0 { + return nil + } + p = uint64(size) - p + + if err := fw.Write(make([]byte, p)); err != nil { + return errors.Wrap(err, "add padding") + } + return nil +} + +func (fw *FileWriter) Close() error { + if err := fw.Flush(); err != nil { + return err + } + if err := fw.f.Sync(); err != nil { + return err + } + return fw.f.Close() +} + +func (fw *FileWriter) Remove() error { + return os.Remove(fw.name) +} + +func (w *binaryWriter) AddIndexMeta(indexVersion int, indexPostingOffsetTable uint64) error { + w.buf.Reset() + w.buf.PutByte(byte(indexVersion)) + w.buf.PutBE64(indexPostingOffsetTable) + return w.f.Write(w.buf.Get()) +} + +func (w *binaryWriter) SymbolsWriter() io.Writer { + w.toc.Symbols = w.f.Pos() + return w +} + +func (w *binaryWriter) PostingOffsetsWriter() io.Writer { + w.toc.PostingsOffsetTable = w.f.Pos() + return w +} + +func (w *binaryWriter) WriteTOC() error { + w.buf.Reset() + + w.buf.PutBE64(w.toc.Symbols) + w.buf.PutBE64(w.toc.PostingsOffsetTable) + + w.buf.PutHash(w.crc32) + + return w.f.Write(w.buf.Get()) +} + +func (w *binaryWriter) Write(p []byte) (int, error) { + n := w.f.Pos() + err := w.f.Write(p) + return int(w.f.Pos() - n), err +} + +func (w *binaryWriter) Close() error { + return w.f.Close() +} + +type postingValueOffsets struct { + offsets []postingOffset + lastValOffset int64 +} + +type postingOffset struct { + // label value. + value string + // offset of this entry in posting offset table in index-header file. + tableOff int +} + +type BinaryReader struct { + b index.ByteSlice + toc *BinaryTOC + + // Close that releases the underlying resources of the byte slice. + c io.Closer + + // Map of LabelName to a list of some LabelValues's position in the offset table. + // The first and last values for each name are always present. + postings map[string]*postingValueOffsets + // For the v1 format, labelname -> labelvalue -> offset. + postingsV1 map[string]map[string]index.Range + + symbols *index.Symbols + nameSymbols map[uint32]string // Cache of the label name symbol lookups, + // as there are not many and they are half of all lookups. + + dec *index.Decoder + + version int + indexVersion int + indexLastPostingEnd int64 +} + +// NewBinaryReader loads or builds new index-header if not present on disk. +func NewBinaryReader(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID) (*BinaryReader, error) { + binfn := filepath.Join(dir, id.String(), block.IndexHeaderFilename) + br, err := newFileBinaryReader(binfn) + if err == nil { + return br, nil + } + + level.Debug(logger).Log("msg", "failed to read index-header from disk; recreating", "path", binfn, "err", err) + + start := time.Now() + if err := WriteBinary(ctx, bkt, id, binfn); err != nil { + return nil, errors.Wrap(err, "write index header") + } + + level.Debug(logger).Log("msg", "built index-header file", "path", binfn, "elapsed", time.Since(start)) + + return newFileBinaryReader(binfn) +} + +func newFileBinaryReader(path string) (bw *BinaryReader, err error) { + f, err := fileutil.OpenMmapFile(path) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + runutil.CloseWithErrCapture(&err, f, "index header close") + } + }() + + r := &BinaryReader{ + b: realByteSlice(f.Bytes()), + c: f, + postings: map[string]*postingValueOffsets{}, + } + + // Verify header. + if r.b.Len() < headerLen { + return nil, errors.Wrap(encoding.ErrInvalidSize, "index header's header") + } + if m := binary.BigEndian.Uint32(r.b.Range(0, 4)); m != MagicIndex { + return nil, errors.Errorf("invalid magic number %x", m) + } + r.version = int(r.b.Range(4, 5)[0]) + r.indexVersion = int(r.b.Range(5, 6)[0]) + + r.indexLastPostingEnd = int64(binary.BigEndian.Uint64(r.b.Range(6, headerLen))) + + if r.version != BinaryFormatV1 { + return nil, errors.Errorf("unknown index header file version %d", r.version) + } + + r.toc, err = newBinaryTOCFromByteSlice(r.b) + if err != nil { + return nil, errors.Wrap(err, "read index header TOC") + } + + r.symbols, err = index.NewSymbols(r.b, r.indexVersion, int(r.toc.Symbols)) + if err != nil { + return nil, errors.Wrap(err, "read symbols") + } + + var lastKey []string + if r.indexVersion == index.FormatV1 { + // Earlier V1 formats don't have a sorted postings offset table, so + // load the whole offset table into memory. + r.postingsV1 = map[string]map[string]index.Range{} + + var prevRng index.Range + if err := index.ReadOffsetTable(r.b, r.toc.PostingsOffsetTable, func(key []string, off uint64, _ int) error { + if len(key) != 2 { + return errors.Errorf("unexpected key length for posting table %d", len(key)) + } + + if lastKey != nil { + prevRng.End = int64(off - crc32.Size) + r.postingsV1[lastKey[0]][lastKey[1]] = prevRng + } + + if _, ok := r.postingsV1[key[0]]; !ok { + r.postingsV1[key[0]] = map[string]index.Range{} + r.postings[key[0]] = nil // Used to get a list of labelnames in places. + } + + lastKey = key + prevRng = index.Range{Start: int64(off + postingLengthFieldSize)} + return nil + }); err != nil { + return nil, errors.Wrap(err, "read postings table") + } + if lastKey != nil { + prevRng.End = r.indexLastPostingEnd - crc32.Size + r.postingsV1[lastKey[0]][lastKey[1]] = prevRng + } + } else { + lastTableOff := 0 + valueCount := 0 + + // For the postings offset table we keep every label name but only every nth + // label value (plus the first and last one), to save memory. + if err := index.ReadOffsetTable(r.b, r.toc.PostingsOffsetTable, func(key []string, off uint64, tableOff int) error { + if len(key) != 2 { + return errors.Errorf("unexpected key length for posting table %d", len(key)) + } + + if _, ok := r.postings[key[0]]; !ok { + // Next label name. + r.postings[key[0]] = &postingValueOffsets{} + if lastKey != nil { + if valueCount%symbolFactor != 0 { + // Always include last value for each label name. + r.postings[lastKey[0]].offsets = append(r.postings[lastKey[0]].offsets, postingOffset{value: lastKey[1], tableOff: lastTableOff}) + } + r.postings[lastKey[0]].lastValOffset = int64(off - crc32.Size) + lastKey = nil + } + valueCount = 0 + } + + lastKey = key + if valueCount%symbolFactor == 0 { + r.postings[key[0]].offsets = append(r.postings[key[0]].offsets, postingOffset{value: key[1], tableOff: tableOff}) + return nil + } + + lastTableOff = tableOff + valueCount++ + return nil + }); err != nil { + return nil, errors.Wrap(err, "read postings table") + } + if lastKey != nil { + if valueCount%symbolFactor != 0 { + r.postings[lastKey[0]].offsets = append(r.postings[lastKey[0]].offsets, postingOffset{value: lastKey[1], tableOff: lastTableOff}) + } + r.postings[lastKey[0]].lastValOffset = r.indexLastPostingEnd - crc32.Size + } + // Trim any extra space in the slices. + for k, v := range r.postings { + l := make([]postingOffset, len(v.offsets)) + copy(l, v.offsets) + r.postings[k].offsets = l + } + } + + r.nameSymbols = make(map[uint32]string, len(r.postings)) + for k := range r.postings { + if k == "" { + continue + } + off, err := r.symbols.ReverseLookup(k) + if err != nil { + return nil, errors.Wrap(err, "reverse symbol lookup") + } + r.nameSymbols[off] = k + } + + r.dec = &index.Decoder{LookupSymbol: r.LookupSymbol} + + return r, nil +} + +// newBinaryTOCFromByteSlice return parsed TOC from given index header byte slice. +func newBinaryTOCFromByteSlice(bs index.ByteSlice) (*BinaryTOC, error) { + if bs.Len() < binaryTOCLen { + return nil, encoding.ErrInvalidSize + } + b := bs.Range(bs.Len()-binaryTOCLen, bs.Len()) + + expCRC := binary.BigEndian.Uint32(b[len(b)-4:]) + d := encoding.Decbuf{B: b[:len(b)-4]} + + if d.Crc32(castagnoliTable) != expCRC { + return nil, errors.Wrap(encoding.ErrInvalidChecksum, "read index header TOC") + } + + if err := d.Err(); err != nil { + return nil, err + } + + return &BinaryTOC{ + Symbols: d.Be64(), + PostingsOffsetTable: d.Be64(), + }, nil +} + +func (r BinaryReader) IndexVersion() int { + return r.indexVersion +} + +// TODO(bwplotka): Get advantage of multi value offset fetch. +func (r BinaryReader) PostingsOffset(name string, value string) (index.Range, error) { + rngs, err := r.postingsOffset(name, value) + if err != nil { + return index.Range{}, err + } + if len(rngs) != 1 { + return index.Range{}, NotFoundRangeErr + } + return rngs[0], nil +} + +func (r BinaryReader) postingsOffset(name string, values ...string) ([]index.Range, error) { + rngs := make([]index.Range, 0, len(values)) + if r.indexVersion == index.FormatV1 { + e, ok := r.postingsV1[name] + if !ok { + return nil, nil + } + for _, v := range values { + rng, ok := e[v] + if !ok { + continue + } + rngs = append(rngs, rng) + } + return rngs, nil + } + + e, ok := r.postings[name] + if !ok { + return nil, nil + } + + if len(values) == 0 { + return nil, nil + } + + skip := 0 + valueIndex := 0 + for valueIndex < len(values) && values[valueIndex] < e.offsets[0].value { + // Discard values before the start. + valueIndex++ + } + + var tmpRngs []index.Range // The start, end offsets in the postings table in the original index file. + for valueIndex < len(values) { + value := values[valueIndex] + + i := sort.Search(len(e.offsets), func(i int) bool { return e.offsets[i].value >= value }) + if i == len(e.offsets) { + // We're past the end. + break + } + if i > 0 && e.offsets[i].value != value { + // Need to look from previous entry. + i-- + } + // Don't Crc32 the entire postings offset table, this is very slow + // so hope any issues were caught at startup. + d := encoding.NewDecbufAt(r.b, int(r.toc.PostingsOffsetTable), nil) + d.Skip(e.offsets[i].tableOff) + + tmpRngs = tmpRngs[:0] + // Iterate on the offset table. + for d.Err() == nil { + if skip == 0 { + // These are always the same number of bytes, + // and it's faster to skip than parse. + skip = d.Len() + d.Uvarint() // Keycount. + d.UvarintBytes() // Label name. + skip -= d.Len() + } else { + d.Skip(skip) + } + v := d.UvarintBytes() // Label value. + postingOffset := int64(d.Uvarint64()) // Offset. + for string(v) >= value { + if string(v) == value { + tmpRngs = append(tmpRngs, index.Range{Start: postingOffset + postingLengthFieldSize}) + } + valueIndex++ + if valueIndex == len(values) { + break + } + value = values[valueIndex] + } + if i+1 == len(e.offsets) { + for i := range tmpRngs { + tmpRngs[i].End = e.lastValOffset + } + rngs = append(rngs, tmpRngs...) + // Need to go to a later postings offset entry, if there is one. + break + } + + if value >= e.offsets[i+1].value || valueIndex == len(values) { + d.Skip(skip) + d.UvarintBytes() // Label value. + postingOffset := int64(d.Uvarint64()) // Offset. + for j := range tmpRngs { + tmpRngs[j].End = postingOffset - crc32.Size + } + rngs = append(rngs, tmpRngs...) + // Need to go to a later postings offset entry, if there is one. + break + } + } + if d.Err() != nil { + return nil, errors.Wrap(d.Err(), "get postings offset entry") + } + } + + return rngs, nil +} + +func (r BinaryReader) LookupSymbol(o uint32) (string, error) { + if s, ok := r.nameSymbols[o]; ok { + return s, nil + } + + if r.indexVersion == index.FormatV1 { + // For v1 little trick is needed. Refs are actual offset inside index, not index-header. This is different + // of the header length difference between two files. + o += headerLen - index.HeaderLen + } + + return r.symbols.Lookup(o) +} + +func (r BinaryReader) LabelValues(name string) ([]string, error) { + if r.indexVersion == index.FormatV1 { + e, ok := r.postingsV1[name] + if !ok { + return nil, nil + } + values := make([]string, 0, len(e)) + for k := range e { + values = append(values, k) + } + sort.Strings(values) + return values, nil + + } + e, ok := r.postings[name] + if !ok { + return nil, nil + } + if len(e.offsets) == 0 { + return nil, nil + } + values := make([]string, 0, len(e.offsets)*symbolFactor) + + d := encoding.NewDecbufAt(r.b, int(r.toc.PostingsOffsetTable), nil) + d.Skip(e.offsets[0].tableOff) + lastVal := e.offsets[len(e.offsets)-1].value + + skip := 0 + for d.Err() == nil { + if skip == 0 { + // These are always the same number of bytes, + // and it's faster to skip than parse. + skip = d.Len() + d.Uvarint() // Keycount. + d.UvarintBytes() // Label name. + skip -= d.Len() + } else { + d.Skip(skip) + } + s := yoloString(d.UvarintBytes()) // Label value. + values = append(values, s) + if s == lastVal { + break + } + d.Uvarint64() // Offset. + } + if d.Err() != nil { + return nil, errors.Wrap(d.Err(), "get postings offset entry") + } + return values, nil +} + +func yoloString(b []byte) string { + return *((*string)(unsafe.Pointer(&b))) +} + +func (r BinaryReader) LabelNames() []string { + allPostingsKeyName, _ := index.AllPostingsKey() + labelNames := make([]string, 0, len(r.postings)) + for name := range r.postings { + if name == allPostingsKeyName { + // This is not from any metric. + continue + } + labelNames = append(labelNames, name) + } + sort.Strings(labelNames) + return labelNames +} + +func (r *BinaryReader) Close() error { return r.c.Close() } diff --git a/pkg/block/indexheader/header.go b/pkg/block/indexheader/header.go index 2f3b2fbbcc..04b3f14049 100644 --- a/pkg/block/indexheader/header.go +++ b/pkg/block/indexheader/header.go @@ -1,19 +1,37 @@ package indexheader import ( + "io" + + "github.com/pkg/errors" "github.com/prometheus/prometheus/tsdb/index" ) -// NotFoundRange is a range returned by PostingsOffset when there is no posting for given name and value pairs. -// Has to be default value of index.Range. -var NotFoundRange = index.Range{} +// NotFoundRangeErr is an error returned by PostingsOffset when there is no posting for given name and value pairs. +var NotFoundRangeErr = errors.New("range not found") -// Reader is an interface allowing to read essential, minimal number of index entries from the small portion of index file called header. +// Reader is an interface allowing to read essential, minimal number of index fields from the small portion of index file called header. type Reader interface { + io.Closer + + // IndexVersion returns version of index. IndexVersion() int + + // PostingsOffset returns start and end offsets of postings for given name and value. + // The end offset might be bigger than the actual posting ending, but not larger than the whole index file. + // NotFoundRangeErr is returned when no index can be found for given name and value. // TODO(bwplotka): Move to PostingsOffsets(name string, value ...string) []index.Range and benchmark. - PostingsOffset(name string, value string) index.Range + PostingsOffset(name string, value string) (index.Range, error) + + // LookupSymbol returns string based on given reference. + // Error is return if the symbol can't be found. LookupSymbol(o uint32) (string, error) - LabelValues(name string) []string + + // LabelValues returns all label values for given label name or error. + // If no values are found for label name, or label name does not exists, + // then empty string is returned and no error. + LabelValues(name string) ([]string, error) + + // LabelNames returns all label names. LabelNames() []string } diff --git a/pkg/block/indexheader/header_test.go b/pkg/block/indexheader/header_test.go index 3b360d126b..0fb8af221e 100644 --- a/pkg/block/indexheader/header_test.go +++ b/pkg/block/indexheader/header_test.go @@ -2,14 +2,25 @@ package indexheader import ( "context" + "io" "io/ioutil" + "math" "os" "path/filepath" "testing" "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/oklog/ulid" + "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" + "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/objstore/filesystem" + "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -20,53 +31,365 @@ func TestReaders(t *testing.T) { testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() - b, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, bkt.Close()) }() + + // Create block index version 2. + id1, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, {{Name: "a", Value: "4"}}, - {{Name: "a", Value: "1"}, {Name: "b", Value: "1"}}, - }, 100, 0, 1000, nil, 124) + {{Name: "a", Value: "1"}, {Name: "longer-string", Value: "1"}}, + }, 100, 0, 1000, labels.Labels{{Name: "ext1", Value: "1"}}, 124) testutil.Ok(t, err) - t.Run("JSON", func(t *testing.T) { - fn := filepath.Join(tmpDir, b.String(), "index.cache.json") - testutil.Ok(t, WriteJSON(log.NewNopLogger(), filepath.Join(tmpDir, b.String(), "index"), fn)) + testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(tmpDir, id1.String()))) - jr, err := NewJSONReader(ctx, log.NewNopLogger(), nil, tmpDir, b) - testutil.Ok(t, err) + // Copy block index version 1 for backward compatibility. + /* The block here was produced at the commit + 706602daed1487f7849990678b4ece4599745905 used in 2.0.0 with: + db, _ := Open("v1db", nil, nil, nil) + app := db.Appender() + app.Add(labels.FromStrings("foo", "bar"), 1, 2) + app.Add(labels.FromStrings("foo", "baz"), 3, 4) + app.Add(labels.FromStrings("foo", "meh"), 1000*3600*4, 4) // Not in the block. + // Make sure we've enough values for the lack of sorting of postings offsets to show up. + for i := 0; i < 100; i++ { + app.Add(labels.FromStrings("bar", strconv.FormatInt(int64(i), 10)), 0, 0) + } + app.Commit() + db.compact() + db.Close() + */ + + m, err := metadata.Read("./testdata/index_format_v1") + testutil.Ok(t, err) + testutil.Copy(t, "./testdata/index_format_v1", filepath.Join(tmpDir, m.ULID.String())) + + _, err = metadata.InjectThanos(log.NewNopLogger(), filepath.Join(tmpDir, m.ULID.String()), metadata.Thanos{ + Labels: labels.Labels{{Name: "ext1", Value: "1"}}.Map(), + Downsample: metadata.ThanosDownsample{Resolution: 0}, + Source: metadata.TestSource, + }, &m.BlockMeta) + testutil.Ok(t, err) + testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(tmpDir, m.ULID.String()))) + + for _, id := range []ulid.ULID{id1, m.ULID} { + t.Run(id.String(), func(t *testing.T) { + indexFile, err := fileutil.OpenMmapFile(filepath.Join(tmpDir, id.String(), block.IndexFilename)) + testutil.Ok(t, err) + defer func() { _ = indexFile.Close() }() + + b := realByteSlice(indexFile.Bytes()) + + t.Run("binary", func(t *testing.T) { + fn := filepath.Join(tmpDir, id.String(), block.IndexHeaderFilename) + testutil.Ok(t, WriteBinary(ctx, bkt, id, fn)) + + br, err := NewBinaryReader(ctx, log.NewNopLogger(), nil, tmpDir, id) + testutil.Ok(t, err) + + defer func() { testutil.Ok(t, br.Close()) }() - testutil.Equals(t, 6, len(jr.symbols)) - testutil.Equals(t, 2, len(jr.lvals)) - testutil.Equals(t, 6, len(jr.postings)) + if id == id1 { + testutil.Equals(t, 1, br.version) + testutil.Equals(t, 2, br.indexVersion) + testutil.Equals(t, &BinaryTOC{Symbols: headerLen, PostingsOffsetTable: 50}, br.toc) + testutil.Equals(t, int64(330), br.indexLastPostingEnd) + testutil.Equals(t, 8, br.symbols.Size()) + testutil.Equals(t, 3, len(br.postings)) + testutil.Equals(t, 0, len(br.postingsV1)) + testutil.Equals(t, 2, len(br.nameSymbols)) + } + + compareIndexToHeader(t, b, br) + }) + + t.Run("json", func(t *testing.T) { + fn := filepath.Join(tmpDir, id.String(), block.IndexCacheFilename) + testutil.Ok(t, WriteJSON(log.NewNopLogger(), filepath.Join(tmpDir, id.String(), "index"), fn)) + + jr, err := NewJSONReader(ctx, log.NewNopLogger(), nil, tmpDir, id) + testutil.Ok(t, err) + + defer func() { testutil.Ok(t, jr.Close()) }() + + if id == id1 { + testutil.Equals(t, 6, len(jr.symbols)) + testutil.Equals(t, 2, len(jr.lvals)) + testutil.Equals(t, 6, len(jr.postings)) + } + + compareIndexToHeader(t, b, jr) + }) + }) + } - testReader(t, jr) - }) } -func testReader(t *testing.T, r Reader) { - testutil.Equals(t, 2, r.IndexVersion()) - exp := []string{"1", "2", "3", "4", "a", "b"} - for i := range exp { - r, err := r.LookupSymbol(uint32(i)) +func compareIndexToHeader(t *testing.T, indexByteSlice index.ByteSlice, headerReader Reader) { + indexReader, err := index.NewReader(indexByteSlice) + testutil.Ok(t, err) + defer func() { _ = indexReader.Close() }() + + testutil.Equals(t, indexReader.Version(), headerReader.IndexVersion()) + + if indexReader.Version() == index.FormatV2 { + // For v2 symbols ref sequential integers 0, 1, 2 etc. + iter := indexReader.Symbols() + i := 0 + for iter.Next() { + r, err := headerReader.LookupSymbol(uint32(i)) + testutil.Ok(t, err) + testutil.Equals(t, iter.At(), r) + + i++ + } + testutil.Ok(t, iter.Err()) + _, err := headerReader.LookupSymbol(uint32(i)) + testutil.NotOk(t, err) + + } else { + // For v1 symbols refs are actual offsets in the index. + symbols, err := getSymbolTable(indexByteSlice) testutil.Ok(t, err) - testutil.Equals(t, exp[i], r) + + for refs, sym := range symbols { + r, err := headerReader.LookupSymbol(refs) + testutil.Ok(t, err) + testutil.Equals(t, sym, r) + } + _, err = headerReader.LookupSymbol(200000) + testutil.NotOk(t, err) + } + + expLabelNames, err := indexReader.LabelNames() + testutil.Ok(t, err) + testutil.Equals(t, expLabelNames, headerReader.LabelNames()) + + expRanges, err := indexReader.PostingsRanges() + testutil.Ok(t, err) + + minStart := int64(math.MaxInt64) + maxEnd := int64(math.MinInt64) + for il, lname := range expLabelNames { + expectedLabelVals, err := indexReader.LabelValues(lname) + testutil.Ok(t, err) + + vals, err := headerReader.LabelValues(lname) + testutil.Ok(t, err) + testutil.Equals(t, expectedLabelVals, vals) + + for iv, v := range vals { + if minStart > expRanges[labels.Label{Name: lname, Value: v}].Start { + minStart = expRanges[labels.Label{Name: lname, Value: v}].Start + } + if maxEnd < expRanges[labels.Label{Name: lname, Value: v}].End { + maxEnd = expRanges[labels.Label{Name: lname, Value: v}].End + } + + ptr, err := headerReader.PostingsOffset(lname, v) + testutil.Ok(t, err) + + // For index-cache those values are exact. + // + // For binary they are exact except last item posting offset. It's good enough if the value is larger than exact posting ending. + if indexReader.Version() == index.FormatV2 { + if iv == len(vals)-1 && il == len(expLabelNames)-1 { + testutil.Equals(t, expRanges[labels.Label{Name: lname, Value: v}].Start, ptr.Start) + testutil.Assert(t, expRanges[labels.Label{Name: lname, Value: v}].End <= ptr.End, "got offset %v earlier than actual posting end %v ", ptr.End, expRanges[labels.Label{Name: lname, Value: v}].End) + continue + } + } else { + // For index formatV1 the last one does not mean literally last value, as postings were not sorted. + // Account for that. We know it's 40 label value. + if v == "40" { + testutil.Equals(t, expRanges[labels.Label{Name: lname, Value: v}].Start, ptr.Start) + testutil.Assert(t, expRanges[labels.Label{Name: lname, Value: v}].End <= ptr.End, "got offset %v earlier than actual posting end %v ", ptr.End, expRanges[labels.Label{Name: lname, Value: v}].End) + continue + } + } + testutil.Equals(t, expRanges[labels.Label{Name: lname, Value: v}], ptr) + } } - _, err := r.LookupSymbol(uint32(len(exp))) + + ptr, err := headerReader.PostingsOffset(index.AllPostingsKey()) + testutil.Ok(t, err) + testutil.Equals(t, expRanges[labels.Label{Name: "", Value: ""}].Start, ptr.Start) + testutil.Equals(t, expRanges[labels.Label{Name: "", Value: ""}].End, ptr.End) + + vals, err := indexReader.LabelValues("not-existing") + testutil.Ok(t, err) + testutil.Equals(t, []string(nil), vals) + + _, err = headerReader.PostingsOffset("not-existing", "1") testutil.NotOk(t, err) +} - testutil.Equals(t, []string{"1", "2", "3", "4"}, r.LabelValues("a")) - testutil.Equals(t, []string{"1"}, r.LabelValues("b")) - testutil.Equals(t, []string{}, r.LabelValues("c")) +func prepareIndexV2Block(t testing.TB, tmpDir string, bkt objstore.Bucket) *metadata.Meta { + /* Copy index 6MB block index version 2. It was generated via thanosbench. Meta.json: + { + "ulid": "01DRBP4RNVZ94135ZA6B10EMRR", + "minTime": 1570766415000, + "maxTime": 1570939215001, + "stats": { + "numSamples": 115210000, + "numSeries": 10000, + "numChunks": 990000 + }, + "compaction": { + "level": 1, + "sources": [ + "01DRBP4RNVZ94135ZA6B10EMRR" + ] + }, + "version": 1, + "thanos": { + "labels": { + "cluster": "one", + "dataset": "continuous" + }, + "downsample": { + "resolution": 0 + }, + "source": "blockgen" + } + } + */ - testutil.Equals(t, []string{"a", "b"}, r.LabelNames()) + m, err := metadata.Read("./testdata/index_format_v2") + testutil.Ok(t, err) + testutil.Copy(t, "./testdata/index_format_v2", filepath.Join(tmpDir, m.ULID.String())) - ptr := r.PostingsOffset("a", "1") - testutil.Equals(t, index.Range{Start: 200, End: 212}, ptr) + _, err = metadata.InjectThanos(log.NewNopLogger(), filepath.Join(tmpDir, m.ULID.String()), metadata.Thanos{ + Labels: labels.Labels{{Name: "ext1", Value: "1"}}.Map(), + Downsample: metadata.ThanosDownsample{Resolution: 0}, + Source: metadata.TestSource, + }, &m.BlockMeta) + testutil.Ok(t, err) + testutil.Ok(t, block.Upload(context.Background(), log.NewNopLogger(), bkt, filepath.Join(tmpDir, m.ULID.String()))) - ptr = r.PostingsOffset("a", "2") - testutil.Equals(t, index.Range{Start: 220, End: 228}, ptr) + return m +} - ptr = r.PostingsOffset("b", "2") - testutil.Equals(t, NotFoundRange, ptr) +func BenchmarkJSONWrite(t *testing.B) { + ctx := context.Background() + logger := log.NewNopLogger() + tmpDir, err := ioutil.TempDir("", "bench-indexheader") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() + + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, bkt.Close()) }() + + m := prepareIndexV2Block(t, tmpDir, bkt) + testutil.Ok(t, os.MkdirAll(filepath.Join(tmpDir, "local", m.ULID.String()), os.ModePerm)) + fn := filepath.Join(tmpDir, m.ULID.String(), block.IndexCacheFilename) + t.ResetTimer() + for i := 0; i < t.N; i++ { + testutil.Ok(t, forceDownloadFile( + ctx, + logger, + bkt, + filepath.Join(m.ULID.String(), block.IndexFilename), + filepath.Join(tmpDir, "local", m.ULID.String(), block.IndexFilename), + )) + testutil.Ok(t, WriteJSON(logger, filepath.Join(tmpDir, "local", m.ULID.String(), block.IndexFilename), fn)) + } +} + +func forceDownloadFile(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, src, dst string) (err error) { + rc, err := bkt.Get(ctx, src) + if err != nil { + return errors.Wrapf(err, "get file %s", src) + } + defer runutil.CloseWithLogOnErr(logger, rc, "download block's file reader") + + f, err := os.OpenFile(dst, os.O_CREATE|os.O_RDWR, os.ModePerm) + if err != nil { + return errors.Wrap(err, "create file") + } + + if _, err := f.Seek(0, 0); err != nil { + return err + } + defer func() { + if err != nil { + if rerr := os.Remove(dst); rerr != nil { + level.Warn(logger).Log("msg", "failed to remove partially downloaded file", "file", dst, "err", rerr) + } + } + }() + defer runutil.CloseWithLogOnErr(logger, f, "download block's output file") + + if _, err = io.Copy(f, rc); err != nil { + return errors.Wrap(err, "copy object to file") + } + return nil +} + +func BenchmarkJSONReader(t *testing.B) { + logger := log.NewNopLogger() + tmpDir, err := ioutil.TempDir("", "bench-indexheader") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() + + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, bkt.Close()) }() + + m := prepareIndexV2Block(t, tmpDir, bkt) + fn := filepath.Join(tmpDir, m.ULID.String(), block.IndexCacheFilename) + testutil.Ok(t, WriteJSON(log.NewNopLogger(), filepath.Join(tmpDir, m.ULID.String(), block.IndexFilename), fn)) + + t.ResetTimer() + for i := 0; i < t.N; i++ { + jr, err := newFileJSONReader(logger, fn) + testutil.Ok(t, err) + testutil.Ok(t, jr.Close()) + } +} + +func BenchmarkBinaryWrite(t *testing.B) { + ctx := context.Background() + + tmpDir, err := ioutil.TempDir("", "bench-indexheader") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() + + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, bkt.Close()) }() + + m := prepareIndexV2Block(t, tmpDir, bkt) + fn := filepath.Join(tmpDir, m.ULID.String(), block.IndexHeaderFilename) + + t.ResetTimer() + for i := 0; i < t.N; i++ { + testutil.Ok(t, WriteBinary(ctx, bkt, m.ULID, fn)) + } +} + +func BenchmarkBinaryReader(t *testing.B) { + ctx := context.Background() + tmpDir, err := ioutil.TempDir("", "bench-indexheader") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() + + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(t, err) + + m := prepareIndexV2Block(t, tmpDir, bkt) + fn := filepath.Join(tmpDir, m.ULID.String(), block.IndexHeaderFilename) + testutil.Ok(t, WriteBinary(ctx, bkt, m.ULID, fn)) + + t.ResetTimer() + for i := 0; i < t.N; i++ { + br, err := newFileBinaryReader(fn) + testutil.Ok(t, err) + testutil.Ok(t, br.Close()) + } } diff --git a/pkg/block/indexheader/json_reader.go b/pkg/block/indexheader/json_reader.go index cf0277a929..67b502e791 100644 --- a/pkg/block/indexheader/json_reader.go +++ b/pkg/block/indexheader/json_reader.go @@ -3,7 +3,6 @@ package indexheader import ( "context" "encoding/json" - "hash/crc32" "io/ioutil" "os" "path/filepath" @@ -58,15 +57,6 @@ func (b realByteSlice) Sub(start, end int) index.ByteSlice { return b[start:end] } -// The table gets initialized with sync.Once but may still cause a race -// with any other use of the crc32 package anywhere. Thus we initialize it -// before. -var castagnoliTable *crc32.Table - -func init() { - castagnoliTable = crc32.MakeTable(crc32.Castagnoli) -} - // readSymbols reads the symbol table fully into memory and allocates proper strings for them. // Strings backed by the mmap'd memory would cause memory faults if applications keep using them // after the reader is closed. @@ -126,7 +116,6 @@ func getSymbolTable(b index.ByteSlice) (map[uint32]string, error) { for o, s := range symbolsV2 { symbolsTable[uint32(o)] = s } - return symbolsTable, nil } @@ -206,11 +195,12 @@ type JSONReader struct { postings map[labels.Label]index.Range } +// NewJSONReader loads or builds new index-cache.json if not present on disk or object storage. func NewJSONReader(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID) (*JSONReader, error) { cachefn := filepath.Join(dir, id.String(), block.IndexCacheFilename) - jr, err := newJSONReaderFromFile(logger, cachefn) + jr, err := newFileJSONReader(logger, cachefn) if err == nil { - return jr, err + return jr, nil } if !os.IsNotExist(errors.Cause(err)) && errors.Cause(err) != jsonUnmarshalError { @@ -219,7 +209,7 @@ func NewJSONReader(ctx context.Context, logger log.Logger, bkt objstore.BucketRe // Try to download index cache file from object store. if err = objstore.DownloadFile(ctx, logger, bkt, filepath.Join(id.String(), block.IndexCacheFilename), cachefn); err == nil { - return newJSONReaderFromFile(logger, cachefn) + return newFileJSONReader(logger, cachefn) } if !bkt.IsObjNotFoundErr(errors.Cause(err)) && errors.Cause(err) != jsonUnmarshalError { @@ -243,16 +233,16 @@ func NewJSONReader(ctx context.Context, logger log.Logger, bkt objstore.BucketRe return nil, errors.Wrap(err, "write index cache") } - return newJSONReaderFromFile(logger, cachefn) + return newFileJSONReader(logger, cachefn) } // ReadJSON reads an index cache file. -func newJSONReaderFromFile(logger log.Logger, fn string) (*JSONReader, error) { +func newFileJSONReader(logger log.Logger, fn string) (*JSONReader, error) { f, err := os.Open(fn) if err != nil { return nil, errors.Wrap(err, "open file") } - defer runutil.CloseWithLogOnErr(logger, f, "index reader") + defer runutil.CloseWithLogOnErr(logger, f, "index cache json close") var v indexCache @@ -317,20 +307,29 @@ func (r *JSONReader) IndexVersion() int { func (r *JSONReader) LookupSymbol(o uint32) (string, error) { idx := int(o) if idx >= len(r.symbols) { - return "", errors.Errorf("bucketIndexReader: unknown symbol offset %d", o) + return "", errors.Errorf("indexJSONReader: unknown symbol offset %d", o) } - + // NOTE: This is not entirely correct, symbols slice can have gaps. Not fixing as JSON reader + // is replaced by index-header. return r.symbols[idx], nil } -func (r *JSONReader) PostingsOffset(name, value string) index.Range { - return r.postings[labels.Label{Name: name, Value: value}] +func (r *JSONReader) PostingsOffset(name, value string) (index.Range, error) { + rng, ok := r.postings[labels.Label{Name: name, Value: value}] + if !ok { + return index.Range{}, NotFoundRangeErr + } + return rng, nil } // LabelValues returns label values for single name. -func (r *JSONReader) LabelValues(name string) []string { - res := make([]string, 0, len(r.lvals[name])) - return append(res, r.lvals[name]...) +func (r *JSONReader) LabelValues(name string) ([]string, error) { + vals, ok := r.lvals[name] + if !ok { + return nil, nil + } + res := make([]string, 0, len(vals)) + return append(res, vals...), nil } // LabelNames returns a list of label names. @@ -342,3 +341,5 @@ func (r *JSONReader) LabelNames() []string { sort.Strings(res) return res } + +func (r *JSONReader) Close() error { return nil } diff --git a/pkg/block/indexheader/testdata/index_format_v1/chunks/.gitkeep b/pkg/block/indexheader/testdata/index_format_v1/chunks/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pkg/block/indexheader/testdata/index_format_v1/index b/pkg/block/indexheader/testdata/index_format_v1/index new file mode 100644 index 0000000000000000000000000000000000000000..76b0a309299da014535d2386879e4ad70688f775 GIT binary patch literal 5407 zcmYkA2~<_p8prS1_uR`M;GBvR4$IOU?wQXqwbGZSn5kKosmmdG&PiI9sgV+*@thO% zp^+mhhV~qYa7fLrERq~j7Euu>qT-kXDdqcr+Q)Q+qOoeJp)tPEA)nuwAXiW1lO=6nN zG=*s@)6k+b&0w0zG)vH#?qj;dbeZW2(^aNxOh==^bd%|pU@*hS42c;sGZbd1%+Q#j zGXw1=Gc3VmrjMBtGi7Ef%v71FF;i!z!Az7a!D5z=SrW5kW+}{4nWZsHXO_V%lUbH~ z+}L-(9CzH?Z@Z^WnE3G%Tl$9z*(?%s7+RRWD1@XgBzn+)ZhlWvAJTcj^#LkLBS^2E zC7pMZG=U86I%Ky%5)BzwmitORNlVDAxyrn-Fh~p}NjTQyUwg>f!P0~t8>+^Tb%5;< zyn|PcC+i5y6ZSVgKaQ+3?36IGVEP2IZm?3JAm*GwCc*9qEzj=wflP%ZvQK_YC?hjq zYuNUxCBwyU1T$!5bgdgfJ} z2ql{X+v|CvTiblHd9V|n&^g_s!eI#lw8&$&|L9xNg``(KmpeUmmo$mA#xwWW*wLhm zNtb#z_j{%pX$onEx9skwWYT4%UwM1SH#tnYg7mmonH_eBbT#RDZ$+ave$rIZO7GRg zb8AW0kv{as=L>nH>q%2WUOaPPA!!EbhLCsCn;#_IM7lfV`ID`uk!~eD8q!E_(2{gJ z>A8^lcOTCp%_6M``KE>HBF!eP4)KSyeT8%n>5|axX=8Vg?j=nRmE)hPJ^29Xm!a1` z`beW{F6oib==&vqkRBvG6WV&!uEwNCNH2y?pOUzY^cd;=&{H>jIix2@lf!Z^w#X(u zNxClVcGmJ}(gM<~u*Rple??kIdMIpP?9xw2Pm}%_7MH%U_VUk>mW36}nskz?7fA1h zwf!)!Eom`nQh29758osGg)}X^hE1JKT1vVj{Oj-A#gdkj=7kUU-_IbeAUze%j7L9` zULh?FFV6mc0cjQKo$$H^H*5FCP13|VtJL(`OSnzCrcUSfCu%p+UDC`tuMW*VO|N}G znp@}Ih2|Bc4@nE^WGq`!yOI7P{k6`PtB)R2HGx5I)#?4w(@CTYNf$)y&-!&FX%gw` zh%PUr-X~p5x;3I%QuXtsDWo|Otv`IHc72zT=0`j`u6!I-SCE!OoT@IGK)RarM#Nk7 z8rCOGB~6GNVGTV&x{h>ZqM7oL8AK9b$+HTUV zq{Wd}zCKpl)ppXV$W^y~*hSSW(!V47uDXP-4zO${FxT>^Ya7!I9rOgxfj*!w=nn>hm%$J) z3=9V&z$h>Vj0bUG5||3!2Jv7Pm<{HDc_0BS1W8~qNCC^h3a}caf^}d$$N-zbR;(rvE;tB|fMehUI0*_sAvg`rfeWA*`~pfrIj8_vKoz(NZiBnv0eA@h3QViu z#?cGHKm@1@>Vrn035W(QK@4aII)IL#a~H*!ihk+=S|5f(H=yV{Kkf$)4Ya#^$82ay zf0w&h9)9*R`nFvdeE-nr3mg|0yB>@jcx zzrEnNd+p@oLuv2WLKkRvb$6$6xQ=GGJhrFCAy4 zGX}nJ#&PeY;|q?{y7BWpw<{a@A35%I{c=Ag==xdTyIY&2J9Xw-HTSwd$k$EQt;>!J zv)RdaX>aiUg~_`P|6Rv%mif&M*B6e9`=^I={Mm7DtVis=D#(vTKK(|}^$O>QUg^`o zsf+cSCRML-+>`u!uYTPf_rBlZK*D*)-Qm}Ij9L`rKgHD~&{+9+S<1+h&Bg@z{t*BB zv`=~6t_U`+1O0Bhm?5G(;}0s3*{esE(=+&=+~$Nh)PB{G10B*p+dm&9J+B`^|9 z0doMJ10vpE#BBg$5x)cHKsmrQdGK8G)C1Trp01!Lz}$M?1ehDo2Ot@wf~{a5!1Z{} zf--R1=$y>&q=&;|4Wv0wz41l|YO&)zj)3)l;=zFy3U_jhm$Jib3UdnP&x1+`=P z^W}2%Z#%nDqCUJA4Qpw;c&D;r6UJq` zFslEWvqKygx24F_Fz0rinH~GqVE$|uvR+*E?30drgywU$Ee-On6V5fB73BM--|PLl z<7`IR#jmjzc7LAZ*S(*;=D2swq{pe(92dJKw#3%^=Y03ntT#K-{|bD*ggg6Ajli1N z&N>yfi^Y1{E+m#cxS~7m-JAKrq@#|r7N^Eu!hK+$&uY31d1;#Cp49A_dGU_3aScC? z!u?~P?;bsGVLhz5?cU!Dc5Mq9pZkw7rB|>&|Eb^VnpQs6anIR~D|PG#yDqZk$pt~L z7dEVo^#+~K<_}8u2JH_qt?X#iw$AIlt;YU%7x#hPpKx)ZG-sIO-X0mRg}>{#(C6On zok8!h;uKbsxXG!D=WaaAALO{ZL(C2-8`jEc6zR@c`33upUsQjBz z1>ZgR>j#o<76p6e+n;pg&1Yvy+s`lOpHvq-*{_ z609QLWEqt_NmeyanwH!+(4AqX@np%FJSkQVPnzVn+k8cKQYv{;rE2>;U-A?Da+NeY znVCE(S`JT=;SVH1E#gh8QpuB{RP&^n$xZn1q%@vXBQuZ$J%=|*s^3m#5l@;`$&;y9 z+ih}k)7tI?Gc6GCHBqLWj2xaUpPwgPFS4szWmAR^wYK`wTLcEC7+E~YMjlVPRKSy= zmhhxnRd&^?;YsqPv<&p3tLZ$+S{6@|o)<`hQNWuFxr8UvSH+WT)!1!PN~=J3mYU9! zW@PcCYI!`#z5+X`B|J$=6;GC4V^@`w*8C#Pbe;?=iznI4<4IKuc(SY#o;0NjDR9rc zQ^OlgIVC30p{AzuBxzZJBq({jNmmMZGK~^ml&kC}tA;oEQrhsz^QGI#$l^)$<+XV* mZ|8J80O-dPIMBZx^b=D19LJ}Mf`Etm1IKz52Y-0J(fJgCNbR<(~Rl$nndYfyorfvYN{sQBqp(9#a^RPDGDkGi1ea}h?OFs z2qL|xSg?0RvG*Q3zdde$XMeu$I&03FcjkT1K7{+nz0Y0enYGre`RtiCmmxq5B3MD;>A2bZlSg*rC#~sykIWcCK{nQt8;W z(y?2mWA_a@RXS~1>D0c`sY9hx$4aN_|J%9JsY|6(*Gi{ul}_C^=v?W%Wu^>(y7v=bEQi)*tOE7Tcu0) z4Z2pkZdvKtzS6ZrrEAAZ*G`qLohx0tRJvB<-6~zXZ_usMZOclx_LXiOD&0C(x^=2_ z>s;y9rP8fyrCaq&-8bl7>Aq#9d;3cF4wdd5E8RO)x_7R0?^5aBwbH#?rF-{J@4D+A zJrCaVfPH!#wCfSOAGS}ABX-?&qg{7BX!iqq?7Hh`_dNKZBlbJ!$b*kOeAnF%Ib_!Z zdmM4tetT}&e7ODRnsv9|Qi*H+4vN2{67QtMJ1g-nO1!HQ@214NH$MiY{SL|)Iw)i4 zpyg4<&_NkP2W1Q$lreNr#?V0-LkDFHN}C;(F?3YM&`}vfM=ifHhK|Y@Ix1u6sEnbb zGKP-I7?gH8DP!oQjG>bDI%##4F?3SK&`B9XCuIzslrbn}J1b-8tc;Ll>=2Wei=EF?3PJ zprq-ljG?PChOWvOx+-Jns*ItlGKQ|o7`kfxD`V)Yj6tz?Q^wFu8ACT^4BeD5bW_IA zO&LQsWenYvF?7@RrHr9_^Y3eSWenYwF?3hP&|MircV!ISl`(Wz#?W0ELw98i-MjyL zlSP|Wzde2Tf9~C|vRAtvd+#}Y=*q#fcUe7p|BXf+S^eMp<9{FfU!ZIIZG@d36ESVe zTAr+b*OqLai4D1;{r>lM?f=pLpS0Tlz3*M#_Nnvs^yb*GzadYtCS==+=f7>I#IB$- za=`uN$Wyobe}Jp;wr`!cTmEl7Z|_ckrx5v3FYVK3>64_@v}F~iVbqSdB_sQO?*HNK z{&(-U{p!5U?z_Qy-~@2S5jP7~%yGu7)q0k|Y0k*F1Fq{|zp;?O)p*;#&fB*?ktrIP-nrV6SoI=y5EXEVR zQ&JbxiD#|j?RU$*bgks=z&dYBXOz#|zd<;)XD`j3TFhEIZDG2(UP}=>%0v;?R~gy{HMTOSb7NcFh6xn9&N3Er`oC_;+CCSJX&vd z-eRg$*E-&ow!d_cMwYEFP>8I8_ zF>!6JR=0Kv{HmszS<_rgPiUNLE!vXb&l@vnnw@B#YTWU*;>#yLEO|Sk&fADb{JgOi zb%yOEKW{RB5t=>A`%SCJRhp-f<9^-}cJ6rl*(c6DQu20WowqlJyx;tuYQ2$p&7;M& zwOZVgM_WsB?7V3enT;N;Iz1L`$wLn{^Gn`BJ7;EU=BaAO+xITN0#WWEo(zPk~b}b7H8+$uIZuHa%yp|4Xq`4)2c8|&FUWQc>B}cf14+H zJG#!>-H%J&LU*L~z@9BSRb8zg#wBn3Dm&3Sr$0tp^0S}NT9UWW3`W1>?VJDVK3Vd1 zOr5u7!#*J1Q%-<8W!%_9i{IC2`BT+m9_rdDFp_FVT296_%^v8n#Oa~V$D3B2nKRn) zcHaBW{-EUTUv=K*ER?)u^0dfIjGw&4Xb%5mUxH@k`MtxAcr>wQEYB_mrr#w??&EIHoKW_=Ub-Z=#@#c>t zZ~w0I_UaPJ8@bgq`?Q>bB@c6k`n9!;Q~PoH{LU%uHSF zM~bz0t-IYwJL*Yazh5y@L<6)iJ6 zspfRN4gJv^;?}PA=NC?>^LFQ0`6o~<`B^K}&$X?k{k)agRp|6W{gSts-Fd1z-gfxq zj%x^9jkgo)yv&##?Xj*7xiWnSVfKEo$~dO%Er=IQ=nN%c;d{r@*gLvlC049*ef**F4NmkEZ>+ zF`K5f%$d03?Z|6yKUVP82fVGl*3KK5$<#{n;4x>>wdDQA>eNMd!dju)(wpQhw2IO1 zc-v*iDR&6o`hvHIUbXXPtec#@BI??CE24UtIZNej!7JB&N$_?Oc)MoehrRD<~0 zd(5w$g5^`pwU(S|d1FOtc15lIHg|fcwVYa5n=b3w^3*P#Hx3O=e<;^mIOrI^@fkpUh zr_ffq)h1x+zf|6yTfX9Vg17$Qt=Hs_c;8cPr@Tuw596u)%tOsunx?;Y3XE`-7S}u$ zzqXd~BD9}3c22Ez;vwAKZ-cJB^V@>AQ^4EHH^uYDKB?KWB~Fh;Gry*_JhfBMRxz$+ zpof}%OI-6XPpBppe%`d|j58ZM$;2IRug_RAMDTVhczfaPe7v!rwxT&7J8!MEJ9Zn> zgP*t1DjvV%?X?>YxKZ$S8hE?)!jI;9PdNcHpDG@jlXW%CoOa&0ikY=IPek)DPR;zG zTFc|-jjJr$vpdJzbJKp!KY&{M?+XLK+sfPgylMH_n>HDGsOe{%njUI;sOhoje7wa} zWLM0iY5J*oB8*crr{s-aiw~#_nW7mBtM60`5C9? zOc<}70wbDc25S1XcqkiM9<7$NK|xl`hKV-Jm{ zwG)^38=cgRwxiH~^6{2q0*<$@Kl!xPgr=Q!J|HMq~emeua-Egm;w@l7vUy984jm>&u_LXWa z`^?7}Go{d1L3) zT89>oww7@%598EaYtg0h_S_SP{Y>z7CU`r4mgLRy$;(X^Id-fvolGUHme zmL9aVe%`cImWuRL^St}>!f}thEO;9Z-X0pTvG+Y?&Z2dyy<7T>=dH|cV``PdTXp7+ zw>|&xv40BQMu4|vv*US-nTCZ8dQ#Pj=Z#%8vi9?quye=T7O&rQx8Q9gc)NH_JZ}km zZt>>@e&mf)QRJ5`ztp&N$@rry!F1)&zp7%TAX`HWjtPB3(BJa0uzpopBM@;18f@z;y{Z7g_O zG4xZp-qXy&H`^Q4*#@xPdQ=gMw?_zKX%?&r7cdQRNnU9O#p9mo+*yE(8*{ME+SqWZ$)$)nv;{y#~U*ktGn;t z{_yLM$UpCV4tN_s`ZMA^)$;S#ld0)voSGhL&BM5+nMc#w+IZfy%$80p9_H8lmbjfa zts>)^u6E*h8+h3tKO^q9iQsM2GCObF9qX1EKLMW(>d+o)r@&n>8{^chqG^jiv{q{VJ-3#Qzu#J`^U%rr zc}utp$J;+XyMz4mfRn&mpYfaE#;~6P*$(a5vX&lBnDNlM^w)M8lZ{<5vn3}znr2RF ztu8&x>F143O*1ofrW428)}P$&KjJ+-54^p2wd9R?*aK@(TRikrx7ONt-nfg_(mW}x z2g#ddm6pvDcf6f)^Q6BC-X?>$dta8kd2%+riCWKE+>*0QPwf=Stk_z$ByTaDc+S`H zw$&j^zcd&$QeD>5_V ztWxzj-Y%ZKK>mFW7lOAjulaf7Pu8qOO%FBuWSna?&9xS-VTX>l4{iL@ z?FDa_fVbD*%*R`6aS~IPv#p&1Bh)cfT5GM=e{sBJDmvcUKYaE^g10H)?e6nGAN`(Y z>b7{dwAP8RuDsv)Jr6ZkQPV?BPmE4ARX=YDJ9NBVw9%c%3*M%Jx5c+KK5v{nb?B_P zR`SMdbW&>`#v4uhdE@S=bL_(L*89{q7Yp8|fwu{3KOcR+u{ZY2PdzpL7OkyiEiI3> z*3TPPX*#r`<}W(#=S{0_nQdmRP?YyxoX12tu9>?2?&wuE1g11Y-+o^Zhc_Z`G?9HOtKec78 z<_Yab^2W^UikUUdwVGy5%@Z1DKiXP9Z_E(cq2q1Gn|FCz@HPXyJ=^Dt=KD?CktNQ3 zS^PzN{Je!$Z?xU9e?M<@Mry~~=GP7VnBZ+Dc)NA#7X@#Y=~+DNKh)2)sWf}Y#~U-G z>MAr3d!W|x_<6Hr&^(#A<8AG>1I`h=%>r+i-u*?vo7VZd6wk+7=mat+(&Bfmq4_O- zKW|!w9G$=Wo_^=Xy#;TVfwvL;HWj?_9jDfMqsOAPJX)MNbIhUXl$l5J#;;mB(L9VZ zCpA6P7LUcxJbvEje_uJgS@u#PRmUpS*aW;B5|gTXor&%)Du}7^lwk)8cIqvsSC6<@DcgTDDYicBSc5wd}k_ zcH(&JH*fV11aEV}+u{emggmXwTj&I0PQlJwXjd7(<88sHZ;chaT?yXK zKYcU7Tj=iixyba;;++;+OUtkIpv7yapnX+~vpPH1{Pb{bsAe9mmX=fU#%$UuOO=er z@wWE3O?DK#%>!>k=fv}7>0jGdOgyG;+WU>2P{-^}TdU<~oLcgxWwykbIpcS{b(%6` zcfs3L;O+T`C2yHLE#6U~eoxQIYiYgZd+qiyv1}L+KHuBOin*<39CEaUON3u`TKhp zgSXcf#Peq9xd>0_DbQaWZ!tS%hcS7$Hl8U#rdzbOHYTpEWk30NV}>F= zeauhI9Z}OmO%F+vyVh1QPR-02?RZ-|Y;sS*+cNNW;52HP3InQt);Sc)R1p z^}OZdt%zG9lmtr;rSjJMiJj+wxBEA&?6pCUz4x3xbmidLyHvl?y6UX2o1at5=Cpyu ztYywrdGaAvL?_Hygf11g(^vgsC*anu#@jx1-X>gY=dFloq{5aSe>LuSJM6qC|15diug=>`=X}Hbo_cCEvD642J8zB9ZP7jC<4xa@IhrsR0r>bu9!m%JTN=WXHZ>v=1V zJ1eQ_sxcXPq|-e&u6X0)3h_jiqtXHi&(4m z7SEfpTgTfiox1)*-fsuic{}wc$s4~{q&C)V@vIvmkDWJm+sM<3ciOY+7Eud6yAZ>grC-9@SLGf%0!eP`pXebOLt$lWuyd75OZQLdCym2z@Idt0eGiNHz{Ps^eS5dPoEv|Viey)wt zcHUTp9a7Vi(T=x&PTcrX$=l&|-iF+1=Z*O-J{btEbk41Awn%45r zPY>hNe%|P$wq(wD9B<2yn6Omxb_94k={x54l=)L>?vXlGel3sve$z5o;$`kg>n9&? z?A)TO)g5n#%_T=hcx!q>{@N_>__uxajs==@w{mj8zC-tO6o#8WH!!|d8%>8+mCwx z@L!U*qwBog*#Eoc_mp*6ON-N^^+`{tW=?8Lo_xG%RV)+Gy3%rL{?Ir{le=c;S|?hZ zom4%Jw|@*6&A%S$z5jlCOr5tYXG-4K12yxvbuA|!x}&x-89#5KyY~1UZ?DbxD*w8g z_jvnPowo@ONZvd-o8Ccf+FD%8srj`y^H@CeQ^)hhY%$uh3yVkVLGl)pO{-FkJKj2P zf7)l|dwOi0x3#BzFZw<8^w7ko#iPwZi(7JPo_xGnItley{Ms%U*YZf-xGF|xDmva) z|E=F2C2z;odAsj&J8v<)ac!p0ASRh7M(5)#ref&Ym??2>alB=Av=150_E^lCEW4$k0-vcpXBZMI&T*Y`XBM0^3%}>Y!uN$arc{LdKOQr zU6;dKwOhwqzkh9ZtK{v3I&TB#O5S328al~Pf2!V!H zf4;sK{yvAB&)8f%sKx@#9;#&oTK<@|T23uq4scraGUImOBIoab`~8cDp8()$y!8fe zmtU2ZH*J!Uduf(!t(GO;_`Ic>n4LGv+&mt~+eQywzoFo*4|p59KTGw1<7NH$QJ&MIG90b*VC^ymxT`VIen()Tp9 zPtDJXX`awH=NW&$F`K0m?ucu(xFx5>?1Y-{0zKalf4e-kx39n7n27 z(&Bx{6SS=L^QL9f;+Bf7zuH-YnTdJ*-~l zT0d{0op}6?xBq+Y>PdpPe&Fr0Q9o$N_tdkm7VkRC+VXj`bY<~`a$72I&usVFgMzpI z;BD+u$y?+*-o!)bWGvH;@o-;u-nfd|(n+Y_l9L|GgyVVRE<6?8{r32fv%V~NI|aP; zdtUMux+D5qOLHdFT0eH)v<#NGb~;+zk~7rf=Ph9;j<+YieA*L&w^PB}Q)7Ra>pe}_ zXVZPCO){pIwzeGJv>g>4=dS&{CG5oUHg}tswi3LZ2HtL1k&m~8eYVJ35w+rZU1!1D0PuF{OOiLfM;6Vx^lNc?sOi@<{nYeZ=9!N- zR-|TiYR#j?wY8d`acbs~yz#4=W;RW8ZPnv=`{5R=|3~mP5WGz}dy8D}DQmSwosYM+ zyHiqE&@LQr3%0uUCxW-r!Q1eacHXpmLyH%*uO#np`MkB(lt@~kyLP;N@~_v*|9(0M zyuI2OZNQu} zj~BcR1#iP9w>Q71S}mRqwdQYxxSzL1=rC0e<=k)8b9cP`qq6)Z!P}YO?X{cYd9zH0 z98z1>Y91|*7H6K~c(YVzC!snte>`u-ijKF17hc>$@HPy*-T8Joym6|={)67u(458b z#+`=h&|O&a`+2iew0NqWINsjaaQc;kx8dOJ$_qP0zo(X6(_`$viMLQcYf&>#KHfrq znzQ)X6}t`9S{^@dmMYu@b!M&O?a2=w_PpS21b92=wzRxyyUz61#8PX;^YNCd+cKvV z&zoh})g3wBmc72^G{M_Q@Yb()NALZXYBH(vSoUu5v@UO2heb@O9Nwxsa=dNz-S_V+ zcpC-Yp1Q=(n>HCsyol*p>RLRF$(z=B%p{BBEz_0bZS?p7UlhEJ25+nGwDXq9AM_@A z-u-6t&`-@1VLzHj^ZR*YHfm<0ws`0d?Oa>SxTcHaEmP6)*6Y)cY$13X1K#GI)XDpv zW^y*m20fg-rY(M+2>n`|9!p(6Z_K7?R?$3Kw_Iz{%uhe#nvUm<6{#~79dGBqF;V_` znX%w)>U2MET7GSknI4+>WFD?%TwBXH*D@Ze{k*YCXosQx(9Y@C`ea=4#t!LZW@`Ew zr>=S&Z{O{{a#{1W9Ed~@izHm zFKr@t8xP*z=-)Zldm1x=gnMkdKSOi+@3+uw7Jm^Fv*b+6o2R1V?YhMW&k($w1>WwP znU6Qigd4%5&8HmRwCbUgvE*NcfR zKP7mZ2;TaxiRX=XtafTzTst-9JPJ2(OdjTxylI`38E1#o+_e^GpY-^7V`gRu)zymb ze%oxoBYzjXoeSQc8q^i1=|6>3y@l?j2)}=)MO199oPOTeRpj|P-hT9>M?WBVn*`og z&6T|I)4?7XXP>FGCBMaE>Br9-t591qTRfJWoV+Ec=CSjpWw6AnJ94}Y9rL}f3*OEH zZ}T6Oyjk+ccv4L=rUyT7F`0{48#A$Tc&qN(@wWZ)Hz|Jk67(dsB>a@IhcItRLYKuF&3*OELZ~f+X^YW(M{8~J88tk7rshKlXymks& zbyh4wYjw3ad!UDLP5XJ%=BZ`##2s&E{&>!Ag0~C6+e=S$^YZ4&pXApriq5Z{j{SZs zy2GOL`+3vmlcOug+jZAHGgt6-A$YrUc=y71<6fw9?4^nAm^@tT=Z&9wYG$D3j_B9o z77y3PXg_aSX6}xfoikp|<9NIIhf6*#c)JL^&0l2ajkTyPJ!>AuEjj5A)y!Wz1xA=H zG|siuS{{0e(5x$Y<5#J*PUxY|_#JPzJ>N?byq&%LD^H61?Go@ddh}M}Jtf1`o?6YGskSy%T~3)58RB)4|)ktMl=e zsg?VFD|*^mpN+$t)?uo6THdtts>a{_{_U9dy##NUg0~4T$MdH3z=`qq6fxdfx^Z~3 zOfP0iX?fE+d5^c3H@W{!!P^Y*)_20zx!zMx-Q0JbmNO=8juH_S9~Z z?h?FR2HuvO`@iBn)%sx%77zW6rX_D$hpBd&Dv#Eu7MHwf&d9jq?Pn`j{Z{aHIe43V zqvS2JZtk~|b+ylt7Ux=vw)1Amrg>6z%lx4^C2ye_Sly!?Z=Jur!y>`kZ16Vn&Bo-- zv#%sPm&04pyRhV!ylJZs5x{YC&q!qer$J>q1F4#%% zHW$1teMj=fX{6GYsivxB$&+vLsdkYnkA0`4&ax_p$MN=)zwLOV;O$EAcHu=o@xG@y z>gK-diZ})4q^`}M5pAaykG8fE;v_BKr_8tu$J;sEcU>-cn+M(o+%9?J&o5dni-&79 zzZSQw4fW9PpS-0jX0!M$9_F-YOHRofS5aFk(!;e??RfjsHl5_ZUw;*N>(ytQhPmIk zm!h@gt*zf<+QcMpn$r^ZOv&-~<%4@(EAF@X;O)^V>v@yxMNYoO?kt>2BeEfKse0&k-)mAsXC?xD3Zd(8d%#811mI=4)Ko>K3(^H(1z z|2@jZ;O*^u%i)a^2tBDH{N?i&x{Fj3%g0-e>h6A;OdLNgTMkD02rR`Qmr^Grp@+x^>& zxnJ?a-)yj>06?q1{PjXhX2`?2`x;o73LTUJmwQ^=|S>lS(VCD^zPWF)(3OK{?^1mb=U$l)H=<3gwIAxS_*sj7YI>;cA9Sv=XjZ4@S}jfwHT~4| zP!nf6Zp>`atXS0!w{NU{?32K)U5&SW>b%W(EFW*I#oidVXs*?CXdcZkPeF4Q9p_F{ z)#C1$)6N@L<%5IW^A`>IoaWeX(2p;*QwXiXGo{w@XdW%D<%x-> z<;~N%X$J^9jUD96iwttpw8QUBYr8~Q>`v*g?hNwqM4t5EgtHTyfH(J);eUITB~L8Xlohw^F}8tYMTCx zcD&vB$jTQaZ$0b0-MU!vmdVrf2CYl+;&>}!N|rlKt83>?%V3N<-uC<8xzi$7S=7B|`IUH2wH?L8wQ0oU zj9KgFEhclBYb{f?^Tuq{#vQr)?d7LBoh5lYxX#<#%hK|eqt8O`F>O|oH_chZjbyPpLR#KD{WQx}5firbVCT(}tq4!5$>ifLM|H>B zgNJU+zn=WP|Gsc&oww!Ve=Xiq{%jIDNo}nbXMWA2#krPR^D{1a<5xA!Di+PZ_9Mp$ zHhn_bkEJd>k~ek|lbKbj+VOVZep7Fd_uFB0-Y&mJ@|MZdwChHTb8hTSi_;@{v#er< zP(Rm}sU>e(^~fDL-d6PJ(oynuc%8S=uh@Bu?5By_RCz3)O?p_jb_)C|wRR$m(@#xL z(V9EbYDwPMttEq&ITLrhE!us}GRfN!b>8}(vz>TPGkKagX|y=!Ms4X+-fxywjpnh? zB6%xl7ml~JTOY>14*$LH-;S*F_VjgLG z?RY!mte*ViS?}F%$JTj!;rt!Md+NC}oBdgpe)dewo*CEHYJSFJbnO&ao!QtSXE#KW-$64ZyQ}Q@9UDc zMVQ_|LIan_~g%R)_$MRRR&j9GSP@mO|b$x|F}nd*+WiBqrmoaF6< zI&b~%@bhNbbzK7M)04buooI1mb;sM^KR4Hf4;sK{yvB6rsf0A*hCYc z^v7&0)mjOhC9}noYNwVw?1!3bC2;&IH8W7t!B?)}>~CKW}tWGn++M*E-%_-FlCW1aBvSwULnBc8Hc$@j4pSMiTrt@v< zamibmJJQ%Fq6a%~T6L}4Ox*GI+*jB361<%P-bSAO2k(2T^`ONwJtX0?h@9o~#y=dW zO|OW!93svBV5HewdP@*+R_#M)Lg6e zVDaSRjhVHs7_WBdc$>O^-$w**1HjvgA%FC~r`)O51M5oDGwY8SI7LS&n@qD~7gQhbbI^MS1X5?1{Zv(;Gta*OkSSxg@T$@U#>Y;WD>?$U6 z%v$zPgf5OZc3Yi*y* zI_A$G6TF=P-mV$BQz`H138+HNz4KGac&KK6EvKC~Ed%4!>{g3w9-b!GS~PP~`+1`? zv>yq~K7P?c*rzSL~=GSUzamkyO zIWq2e`^>VR^Di8%{eJRc;O*kEf5JyId7Mhv< z4B?KqT{jye@3-OLZQP31S=x>=rhSq>`3#J%G((||9Gb0Z6tVm?5sbV-_s(d7n3t)ZE?I=x=Q8I z7-PIt-hO-D7y1d_MuE4hueI~WU9@#w9B+*_nNoQ>_Uyw)2;N46w`s4g+q`KvNj~0~ zjXH+0RC(9~bLQiX8MMrdSGD{8ZO_g6z9M)V1KtKr+}Zq|X7VTh&bG|_`FJa$L+xZ! z)yl_Pj_QuLO}FaEzW}cG>mg&o+soItE^kFoTbogplnx}ROnTaL2v6+)Pw8K>X;&{t+=y)6O*ulq%_jEjXTX5e0wd#A? zS|`%B?zhk>(I3wnvym~ztJ?9l(FQwjD0n*yyiK^t&KvWyZd=sxytS2S#Peo6U&q^! zZT|Ol!Q0v3t>4>r-i&pVyh=sais!B94pZeXmA6|yHCFz4nF-+S*$e&>{hso_scBQP z+y=};t>uh~`+2i;%j)z~)6-hIINrF^>Up{QZIhoLEq`7(2fW>O>tDRQF()ThG!M*Q zL=S%6SUq&tp=-Ij(6#K1nt3E|{3=&jv}dj3?SsF*WwdxtCxW+Sz5eRu&C)|#^!Rxz zb4TQ@h`N%uBC1q#I^J#@aqVt`w{yYU#h3i|@WzvE9Nt1tpS!TEEtR(|_SsYZeGZer z+mJisc?+FD(SG|WSe&Um+%>h9-{R5MGF}{SnGW6kcG~A}yiMG1=YhB9`~GeHd;0DK zct-1zw)2+ZC-m1D9B=zR^Oy4kZ?bG6U^L!84V$U&aNk!bK;ycv!bo-gY?ak0%J;E&^}8 z9*F0SyVItjeTucUTAcZ{ob*uBUpoax=%luI=r2OE2Wm^La(JtD{|J}?R zr%~o4ScZ1-d^xF9lSmLNLt=3JvV~K&zone&C=coJJNbc%bQlE8h5?{4`e?OfG-o`y4d1HQR?nu-0Q&<1~!={^oaV<~L z@!Bb{6Y9|F+#Pjjo+AA5ycxT4yd8YZ79SMfzs&+~XAIvJB&7Y+^HXN*xrxc5{b_d^ z+HIMWDV4WwpSW{3!P{lv?LP~P<4rpSPFvI3+Ej5rZ|q8|NDnprTAUt>4$VV~CKW}tu+R};UVO-0fDlU2BOsGREau-$Y z-qRmlG)ew@yR*UDEu(fT>^%+bzf6BTZ)J8HQ%mxuW#g@=c|38)+mM^Ls)+mT3h*{} zspKugZPWKujI(ZNoNHrrOkK$vr>$kSc(@}i4?WbD{LE?5cHY=Y4()h5_U&(9BY2ww z-lje;d1Ec^h;dGaaf{|!YOR(fXXx76DR33FR*~Im9>%qtT3lOe@pCP;pEoTtCrUTYFb;%IM*_6(ei$?tcvky-7>Ce)}@DW zO-tUi4BDrOeri^%#vN~8`0f>}1#ee^w_#UH-k4`y)wNU5CcG}~TJl!V?i_Cyow2&7 z;B6jwd-J8%<*l_&y!`vEpi^%8?pn)l=PmLq-u?dVunpv|_g)3wR-e6xcu%#?wRmKo zEly0U8xvpmc+*b7&Rfhpt7{!^dmXb({{2Yv!P~Nx@w`R$*(`7TK7pE3Wt=_Gqs8;_ z#){OeqG^j?>zp|)dFU^dw;5lZBLBYV1>kMkf9$+jroo+t>Z0?rpM1PALtD{y-bk`V zXC~n8x8r|%%`ox(+d}X*`rJLE-_uOpX78&qS+evK&s(GIh`VNQcHXoMjC-`>t=;Bt zpCx!(1m51dp&Z^keKyNy(f)YeitaE~{(QV?)k$*3<9IuC!=2^dAF>#{J@kg;jdv+? zGEPko`_Vk~(_>jnzowaA(~>t!wos2&-I9lEH9zB;-_IK}P;1#damU+FKGV#<54Z%p zt)9GBDeo!!EJBl2OMc0lWmQ}BNZvwcW!$ynt>QN<4l^~?*C@q zLT6?1R~dG^?R3L$B_G_CbhJB6aVVs&=Nc&KL2T&v~x^QL99WX^aT zZz`ksX$q-_IMniqwv`>#z7( zFTvZ@;O(B?dyD%mvToD&NR6Dw&s!sR9@<-}yv;u2)=vrEt^se0r}%j*`jZ$sJ&WJc zv*eAds6#7qEwz@%;)z*n=Pk68jNjdFpZURWPXurGZ&=yueLeQxbNbMggJEM@>}8@gH~7Kra2j>wsca2hdpqotVNF?Pu^#pdrq}e;0z^i{OY=-OXY3O2Yd04%e=?izIEPSnjX)a<$Q}c!I--L&Ae%I z)8fXnbNAc0)km(7_uGDT-fq7)o;UUr*>ls68libY{eIpo*(@GThT5{$;$cor+wZr? zijKFp#{7J+ zwDZPQ)XdDT7_a&rZ{O+t{bs&@JD|?n1+)CTu`V_1GTsQ<(wqH$vt(<8U9$)F9M7AU znenQ2ydAJ-zu(CFt!JIL(^mU=Bg530T21d@TZ#L5Yb!ftpPEj~n|8j{>W;Vn?A`YP z$=iW--d-QDpZ7i0@-xm)6XT7n{k&c{`}i+r6{n zc`I{ztQ9k>w7jvaM$?iv&h04NHj$k>-j3bjv-2cx2iJLb>7Y!Tv|UZevg;%v({oQKX1GNVooPjo|qo|=bNxY$J^#N z9os|RZ->@->wl%cpO~Gt)>_L9{k+9=ZmAxVUt6oymFKHDnc3o@zp5Q?pPhJ1Gyi?z zusUx~J}UdkHCyuw7ADQsFHk=L8y?Wj6$Ylrs~?`f)jQsv39_ZGR8_nX$O7PoZ8 zwbYii^u(-{yfL$;b5wM^{bll`)8zejbe*^R7fIf<9@eF}9JVpES@s^`2 z$J?{pe!ZDr4>_jJ+mdJ2^A^uuj?-znfs1CfRNhvs+>L+y^u1s2{a2m03r8O)-cwF5 zbQ%_asK?@G&la7QF?LAJPP8~Z)S-T^r7n)QOo#4%JK(zS9Vg$@W9z*2UnY6WbyPwyyT7jP-pT7ahRk37W2pCOnbjsCc|#2Eo);ulDC*_p0$p*c^}{Ta>?7j>%84I z?jY}b>dD{41DT}O*3v_*`BTMNt9A;Eq^eSM9<67_spENLHjj3^U4HuxCrjRruk&{8 z)qdXiNz`g-6Xsfan1}Hgt<}ic{{Jb^7j*O?=;webp3fi6Ht^57I*-i3xVx6~96Aq4kPec3H zCK(zJtxJF7^TuvtbXwlDGpcszcpE!lum9)whq!-F{Cz!o;qP;JYi0Ssg^tg&$reA} zTCMfKC7*x;uJ4n3?Sl)h##?Xj_RxRQ@@8Q}`=&vQYs_eI=Aov)Hf`;zmbkW4Z7r*7 zYw6LnB|klBFF38MYTWU5$lHsT3*P#Gx0}vAr1S@sJ(u~LdRe!23Zds7>Sx6=HTx`; zw<9a(zASj_3*Khl*t)!NGO13)lCyRST188SBGzg-W8&rTR-Kr;-=^RE5Pt()?L9pS zyp4EM^2XbkJ8iAIh&_-5YI@ke<|&RhcG3u1@}|usv|G-(svU1PAHDt4;(j|ByuCU3 zQ1PCo>L;}SRDMhUe%?}b(g=AZZv}PbcsqK)zvcID{lMFUH%s0M>L>Tll9qlfj4{u8 z-jYwi@iwt%pU;c?tv`6X`ki>*GWTksT#=bj%|2_>W`x-orw*msk~7A`e%L2-%KME@ zi)J=WSMxaD_BeXQy@Iz>z}te04ioRGmNPWYX@suT{GmBDfAUXv!+y#nZ>eUR$>Vq% zeCe#O3f@iyZxe5i=Z$Ado$05cP9&bUB6gZ8C-c~Oi##vK+hePr8ZCG`4ZNM)=Wz4= z#$81A)*^dF*NW#&n}F827T5B`#96CU-X?AN%FTke0pRWNDSqDgzFQ`s`8fgR(fssK zYkBC=G}ji#8>@%zj%#By>*nK)8K^y7x%+LCd$!_V$W{CGw}Ig8mb>D4<2iczY4tme zz2)Q0vQv7fSzS9p&Cj*emOS}*V--!adR05#hHie;v*La`9lTv}@)73uRIA$-Q6X$q4@`pcLD|j0W-cGvD&YRXx z=3Y(yZZf~tgBH(x2Ag`bGsf!F%*=SGW*%zhVLVjZd1FOtR`kr-@wU|qC-5)CuYLbE z1iZa?%8^OmQ%^rlXU+bpW7aZ{MN8f+tBUYwyAF+Oz1exwG8p5Ix37NjLtBgc?F{gC z>nzEewo_x>X1NN@S+w8ITWBXSQz~L@THZXC_(UX4^ybT3!m#vn(S$6HI+bnOE zoMn3KyoGk9O-74{<`4C!<;_#w@iuJd{k9dnoeAE?4CL=JN_z^eHDS+8nm!9@cWSBF zS{^%ZG3QlX>v-Gvm~PpBkMeyzhJm+tX2NIUR3%9`nlWg11rNZTMqpdE>OT zeQ9y7)pX2S=JfNHstPMwbWB~YrIx%|I?3U2ygjt`(ys{KMuWFkhsxfxT6Aij&^Yhp z(6#h44>dj1wNtP-Egoi$(J{5OwfT6noOPz6 z+fGScL1*E38$IWpZi2V5;BDoTjmaB7JJhLW)%d(|*VH^O%QQ%uf7h&H$(->x-j4s} z4)+P(#(}rlBmO0RKF#E5`Zdx)Ahfu1ZyrIYV`55A@r4i|oqrHf-B1b`rde z2XCVnHzseKSm+!qe#x7qqQzsWUZ%&+Tg>jNYaMTQyj1yv;O#8%_SQ3v$y*U8!g)&G zLMt*`=-M*{_#;;Pd6UHsNy4LZw!?FkEulJq<-mZUvRU~ixYN+P8F;wSypKt#xvUScKZ$g;9nSB`~BqSg12ec_<3XJp*r-F7@D8{ z&|3NKx6mr=FjS|iC3y?&#OQat?S9vmzY+J_B=9!$mEw3a_Sy8kS(*NPyp`D@YsIiv zDsOW-jXXo}b{=?peFAUA;!c58I7w=HG)+G>{fuk6w$sp6TDCIdmY(U6yz#3EJ9qcn zuiyOB*932q!P}G9v3ha5CEQn%`>=I)8gpN9l9oJ@H)cqv9dAGR!0uxNZ|8%zRj)NZ zZ|u-AANr}e*3O%jfpP8J>0xzEYkn<1^Jr`1dDAkpdR05#`fPmLP{G>;;BEe-O>6tL zoPy>lhqsu{wY8yf$y?~0wThY9I^OQxkuR#r$?Z61-gu-rhO?1o57-u9iPlJhWD*U+c3t-n5EZ z=UQCL!#^QnS!?G_t7wT=J8`_7^Xxa|@3*`Jyxn_?O6D; z|1PiyJ*yv0u-b9Rom_xFG2PlC6p;BC^y zCyMtplc(u>{4NMw^bU%(w*3QaxsOJWflk7!Uij0~b@Vy4C^L_kUklR0poc+oj;` z(W!}f(|YCvG*6lFe7uErSoE$f`Ag;Pz|H#)&hj<`ysf;uB;G<#s%ZbZ!JB1DrSkUc zI}Uv?%iB!wHlttfwC|~90`yp38G5KKpKh&I?G%_zb7~&?Q)$+sj>(^oH&&wM?R=v!4KHhS4=y+Q_^vPLS-Yx@gr{2Gg z@D@5BKW`RdLo2qHzf|6S)%mX*XL-9EyghenpVoa(TlW-NtGcCbD(2&PGj`(cx2wMS z`(3lV%?59`UDn2UGw!{`x43o+sj^x4N#zOcIi5GI^Gqj>w{6!vj(>2s_Uj>6fVVjh zHp3ggKcv>`Y97X^nTK&|dNi&18K-8R+9_yXrIQ|NdZ_8KXs)HsoMyA1AT2-hq~(nj zS*5BSZ{6S88rf4jg+}f$Rc~o|^K|8SyJ7zC_sa4% z7rc#~(+qE(oP}~#Cg-ioTj)+>{H5}C={X&4&GL37czfsJ#Jq)0rnT6XylI_l@z&}* zv~H=qP2cO4C)7NgJU!H*^NHt8tEk1f3yV&b$IlyQ$gZ@w=BfG}Z(lg< z$M^%NwO#na&+=Lw~jE6WRN1A$WUh z!Ft~IFP}Z0TG6^x-uis%%YVu8wg|jE^i&JHY4@rYC-d}BTPCdenV;H{lOD-fOg3g_ z=hXC2(?d;9Xm89TdE-}EB~rWl?Th;jd^yY8V(_+VWWRjxX{w&JJX&0viRRbhwNuc( zs#OV%Tk>ljjWNxy^=9YIl3DXqW0R({F1lO%tpWC?dSnpZ=U6CDR?{Yxx~B~du#fcw6*+p-iqi@W21;#l--*_$1bw8!)zizW3BJnKGwp>4%=y<&B+$>d>i{!&`M1j<r1#cRL*wt~F9*aC0NZ_(_HT3bsGwdPM1m%N2m5A~<&+>(dA#q(yVSoJvG zjz8v4_yedl-mV63tIs;6xcAi9XRCZKS38A9?l4tvbWt@GCR zh6duT%oCTq<(P1iZ#;|Mb*a2Pb<*SK3f}gs^Y-}bE%26V8q7m&Ic3eyIJG5b{QQ3f>N= z^LEKiE%2u8REuj<&3td!;!8w7b*lW5H)duuS9*E_u@`YH?9aKU*iY>g*a5!2wUv1b-Fqr3)(ze)(%86EYhb?njrTBhC_h|FT|^${ zm%M2exk~d`;-PsozZTbeki3OvU}mnRu6o@4Htxv>&l0>HQs-^OB{_Lx57d@VoW;YO zMQA&3MN}^{r{t}m`8(cvefr}a1aF7ddAs<|W_YvoWATJewTPTr59>B>oMfrI-8ySb zSHat1b>2=t>2&*h%3WyZSY|x5ANp&jz(}gjEjcY7t)I}fk~b}LWZd0v=l*`%-wWOj zuk-fov}Sk<-BDzpE&kkU$sgMDx_!TKVx{sn=F(ru|9*Nzowu9sX@)m;oB5uwSvKgW z*4AoquBE1*njUHkdy+S1rePLD-f^3X$V$xn~u%`zWmFlu+deR+eg;je#t@7F_) ztn;>{{~-T+YUwS;V_9q5Z?m5TEx+VVtDhs%UZMq^4hs(?d-^wd9Sf5^BfWj!*Rb zjQIZT=sIu19!S6&cc*FYE5^gMp*iWVodUB_(@9NFjAl+vGr#6xT=V;R(^fGvHT~38 zkK=8(_kZjt!P_x)-p(8_SiYy3Jk7pS?15Tao9f2U@>_bKNAkw6QnMnp=3$(g6QGXq zaIL2OylJbrQ)>D%yKubq7;}mIdEsAm-rl*q8QwB^n&mCj6YA%_iqN%FU}j5)^f$8R zUh?r4y1Qyc$J>%kfBb!Mza3lW?f!?F;Vrb5=I5?8j~0(vtL0=|^2V>OOWMy{%$zf8 z9dA2a`TDDZx8v%(Egm#PzNgx>d9Sh-d;z} zH!YhMw^XdIb-X>g>k0CHJHF1_s7KoXZ~N!)KI z)OlNb#u@ha)L6IKpD!A1?RwslPr&hZ-iTekAb2~m&fB9`706r6{aiGk$*0}0pR(fa zx6JuE-d=nEIrG4q`};#S?9mH#bDY_P@6KN3-`^U-0(E(=G7EUzDNd?ksW3Way!0EiF!ur5`_U zS_W33reD+aa~GQ55@$|LOWv4STa_d3-qZV5edO>gZzqAbXGWcAe^1$G=)|=CnKMQ+ zKl3n7T{{JfldGt?R@0iF@gg+)k-V`YotifK9dCy{vq8@+ZzqGdr)Y`h2F8+B#Vf%H)^ex?PX!Dw7fBMqiM-oK^?yP{oAse|B&Tv zAb5M|KLNaH%rj2Sb7!3U4UKcHrdgLB){?yOt6UYTwLGEm(7KvGA8%T>T9vBb@%EJ| zd;C4i+v(u#wsS|=-&3ttBg8evwB96dnln{Av_q|KXr6q$Y1KV($J@TI+_EsEknNA#Uo3H$H|158V!P~euTi{K*SG9P|B)K-T zm!|IzS(ln?sp+AXylEA+Y+78)!|K#p9>$qdTT4H+vr<|m=$0FjAw<4-&Im_I&G@56k>%|S@b=2hIeE*l-E{wi z_RKt*4$W!t*G|DwG1SBAT$?Jcv8vUw^QL8BT+`LvIo{?DesNNkx1r$enRjyX#_6Tf z+9b97p?>DccfYYBH8ZEuT7E6A)wT1cWiZAaZ|B_AvwN1eGr`*p7mbR4PeUhQ>_7LN zY000DH%sOsJTViNylL6AxN%30x3BDR=2x=34Fhj8Zco6Qv7bWu3GE>*Z}G_a3Ta^5^4CtEj~-I|^M(KeblN z&zrU?N8IuDuRk1%zmTiO+X(RX>Xe+k<>+S}zTZ+!gZxm(o_OAv*`pnA zCmeF@o>|^TgSTrhZGksW{$}sAM)PYY63-iVs#T$Fj0lv%yZS|qAe-nLqD?!#H$&H`_D3>;^FPc0L( zcsPwvKi9_SeD_;u#ZyiItx6?oIW{|x;Uhx&Qq{`-v`QZs|5>DRPnt>y`h`+19*Av>w=)P4UpYQu{7 z^^kMG+v!8b+uu{>r{+`{Z)7cbv+Tm+Np*t!C!n;tjB9oMyjjkORj4y-9dDOEa@L~k z{WcN2y>(>*-ZFKY{dHh%8kV*6q{>q}g;d$tVWa7I-n1QMrssJ3`X&dzp5^Ua@bEm_ke%iCn|HteYucuUw%(^)q{&T@F;?Ll4U1SD?- zb>et?bIAGM&hmCXcw0N-Z26vA?i}_^oobR=PAwiiH-5F~T^u#)$U-N^y`<%hnKM&zyq)&VEnc3uT7g#^|)Xc{*{tZFAD;zs~YD1-z{pe~$k>_2kbz?M663KW}ArSahv?yoGjE z^*i4Fe#?I1?|+*L-j-jJfVa@zieQ*IE&WK|EUQ?By6E?jwL-PzEoP^lwT`#VD}NM! zzvVRWHu;sDycPY^S<(6P@m6$K+G%Qa8P~Mr%|a}*S^QOxd^&hL^PGwC z@2NH=W1qR-<(T}Ex0q}(Yg0{6%af0{96NHnop{x%J+j}wT?*b_yDkB5IeIING3}*m z9NsuRjTw4Eb*a2v^0OZe&hj<`yxsTOI?kJRI_2e_82^5i&`7OF}cq@7u z`FP9GmHYnfsm_;(Uk|wqybXA(1>P)Y*A_kNc}qS4$J@+_?QhNAZv{{i=7^kMkqPdpZvet6LSaRmO-z*(wJdU@Iv>SJQmbcm9?Xg>$ z;mxuO%QQ0m9Wnz~F-|`{)K!n;ZL??h z-6YG~72xfbwaxIxJZ({1=2JU`qIYNM#NvtRQ}Py*&9m0=w&!c@zntZ54tTrb;`99P zsV9H4_Z2@E)QymbJ=9Kt9a7U-re+V+F+CK=Tc$(D+v8jRLj3(9bHUq1ceKD;(VxDU zNye-#=YEUnFxA@Pcw>iEP8@IBUb?IJ{k>O$w;_Ee+wZqf_Co#apE^|@=JfMcL`7Do zF1l7)-n4G3amU-Y|9su!+4poFcza=LfxL05)LPG>9{NMIpSRF#jqFd$n`ftvw^#PM zZTl>5SAn)qU+)Zu7=Zbg8_3bj@uKW_ggHLC9TL|8U-QNnlrJ7_+p4uspd}>SPqWxMAp>@;p=Be&@`(4jn zcFyv)2)w;~>IG@vQ*BC&^A(_`r)XU}1y-kKb;dbu#x*}Zp_)0VwLJ7t+j-M6TjJHp zINlEb%;!(f^0pYfJ%3pU|C;4(Id~iVNHe_U zs8#5%_ZHEQpSL0^W=^Nsi4{>-@@DBI)Kl%;@%Hh<<}SbExRtlqn(ZxXAkA@X6Zzmq!#Bc zEIJ==IjTF}-gngSle4^C1Kt)t9>AMrcbdmCd2Ov056ww`?GzZ%sbSM5@wQK$w}A^Al($qTqUA};8_&1VbUD0L=j^VxtG_z>kAk;->%6`2RASz=^9?

3262yd7BQ?WyGfys=j3r=EUl=F~K& zr+Gr-wNubK(c)SsT3pK$+HH)#RNf9fWt(||w}a}u-Tq=u-eM-u2y5;4TO)L<@oed> zRNl@T|JbJmZwJ?Tn|Ib!`+I8HU0d-qCT}rZX_GIGw@kP0J)OJ7;=2WJhtzqScx^Mh zW%4wApQgpRQ+lZBX#|~*H|~hK5q6~YP#kZWitqk<$UncjM(}oMowt)-ZHBi@o+jQx zzZ#*#_FM(ooH(<9&N1__w%Mz42fz8T)My4q(! zi)Z?2kq73XW*#jb>ZzSVs%)%I%}G+zW6|u>l81iD8^0P_-Luy5_U*Ob`>x>ah&pe} zUT=msOAqu|CP06TPF1V8`z>a7mbIJ!wYFCBra85^RwWa6ynSQmi{BQ!9a-n?{PU)_ z?t7~3vFLblyk$p}8nS*2yu;?l4z5lB7cK@vjcq{r- z9Ft#Lo9}+(w5fR_ji9+}$y=%pS*4oO@%G&I``sn(w`1$PE$%g=ocm4NW2WaM-^*#W z7|+KWGibWZ9a(C{^Jb}1^*G)h{@6bk3*L^a^ET;{7I-@W|Ij)$KiyiK9_@5u;>>U9 zfgU?=T*Yp+I6a}7IjJpqI1TzGZ_KQ%Vg_o@TF2XK^LM^c@b>RIZh2tGhkd&LF9mNW)_J?{o)&oHC&*K)#rs5C%ltf%&^XspYjyMS#>~|0 zR;x(AreoH!2j-W&X;qkkn%Sr`c^q#axa$`Db%X6zzx(Iwd*SbMSl)k@d{9}8n#C+} zdZ;aV=%Lo~&|}g0fMa!PR@CD3P}8qz`l;#H;u1Kn%0Sa}anM7}D%4ev1Fp+%XYX3S z;A*_}25*aIHp3h9G_uxaW$D?@TTB&8C#gJIpOUv!6*GApZ@+x)?{l-f^#N~_A83I$ zvQ3?-)%3Yw+~U_fj8kj*H4ptfk=iL(RxyJ`GiRt~ey(MlTJtb2dE-~LZ1iY4v)1wU zwUZZIn&qu8c$+lfGWnh|56_74wy3!mdB5?iF}q_P>X@8d%bc2~U-HJUS~RP8{EoK| z-@D_5S>8?pZ+&OCz#H=$>o$AWF%LC!Qqxm}mb?{FB_?O8X~gr!U3j{3ylvF(sh?zd zI~lw^@=yS8tlQRgju;=ufKJ{sN^K|8S`~C-Z7?|a)A9%ZN(B=O3)RVtyM{Utl zI|a*mrP`fVON+Cv#X~=JKHhRvbiAE;`?5c0dFv0}rp|4FH{;C-fzrI%}%H_j~3@zYKvd;##PkJOid3pJyq>^JE8w6J+r)>0^Tlov<2QW zIh(j@>+!U_HS#nhZv}Phc)R-9-QJhw?NsnKWaw=9p7PGodQ0VDel4dK=h{#$c`Kp{ zyK1fcX?gSP!tr*~tdT>qyqyN#p1Ue1Z=7UX)RMQhxC=|qX?gQhbiDN*`Sm((@BR0z z0pRVnCz|2Sle5|Ply5q9Xr40te%`nX>d+2j*0NSio>F;x{sRlPs{>d2?^y%E+mhi| z$oG^xrS{yV&2q@Kp}Oc=e%?a6ZDfBuZ^pB5_gmj>_QM}|uJLv{c)MUx0B?;vRjoH| zR({^JRjgw1&`%w+mU%2Xo;OzUbnbY2^`?QDf4~0yJqCfd)1FDdo7RICXaB9G{k&Pa z3f*0(-_nD{Qz~!U{&I=nZ7_Iydej{IdurKTTk*v6*0y#^>Wb*R`W>db-%h@Bt9!HG zzYPIzw=FG+H-5)Ut?fP4ZJ^bXylGCY!@JwZ+_miRYk{RcBJ)|mN!p_j<=a>I^UkeT_fX*=@tket0# zwHlK*ZNf3}w7hvbal8$9^X4bAybS|y&%V?QZ`zq=-s>iow793YrnU5FdDssD~Z)47#C*M=8A4}ZBf6%+E<+1c&@zhR%-)m7b zL(C~KCpGh1v}GEa$IhFj6OYI7cG8F+Uy7&Q!qlExK`dGF6kS)|ZqierEWGr~Qe)3iFJ>|_zt@T3>HT^MhuBB%FQ0?c9*;vt{xmME_ z-dHOpkDoV76|FkgR<+~p53gVI+wA=|4!kYBxf$NH{EZy9oFMzR^Tt(eQTutb?8rD< z$J_6|_1hj<-o}Hs3*Sk=8~dTw_QKDF<_V35=4^c4ER(cM$<7-qQd_E2{f@VPOx$@y zmbbIO+rW$F``=Sbtx!)W8}!F$_9=N|W;(SvJr-@rLl3njzvi*?7Fsdmcf1`i>iDO! zyqyi+`rh6QZ(8RuarQ>doLZcoP_5<6#~ZV0njO-^c$vC5-ZHyzyxrLIwQps4n*iQk z>AgVSZ<#!W{+!rawIpvvcM|$SrRvAdTTI3G-qSA*d`B$S%zo(Kt&s2lHy9i~2b+xsOQ*$kK zsl4?Xw999+yiEdc!!K=yH*FVMoV(MsWi36L*7EcI=UOdZ9AjFCTAUqHYyQxzr%p(c<(_hx+Zj zu}Ti@c-wJ@Z?4SpHW|DXC#P8_QM_+r>4iw8&_$XRj4(;7B6Bl zlDE(fEnQV}I^HHTpw`*265O0>xk#X-W-jiv0OVw2odHlR3tmt@~^^0k@WqG?0yiFUp z*#Dj;?59NrLTiQcN&kA@l25?#_Sln$i@$I1BJei+ie`A@xr8#1%3pL3$)}zC)0AqG zrQUBJ?s**k!M57(f4dmGy}70t-cs#U%R`nlt*zDK%%gc2U(Z`@0-EI z7e1DKPcH#)_YGdscHh&?euHipJ8zZ~v3O#-wdB`4rQUC!xUbLHEN@f5+oCJi^R~a8 zy%=UJYc*ypYc)@)y!HRm!$)L!n+o18c&r89VkQu?R-0a`c;he@vm?udOXY3s&SS30 z@-_{;4Loz{y8WJNCt~MK%Mim!=-QYbO6BdZr=8M2%iDDD_QL!C-eP_|kZZN+F;0Kz z=~&kKd1D4mYh7^{^e~SWw|KagwW#TlylGXKLGv(kM!TOE-W>eBbF;i%3f>-gGJv;C z{-E#AW7cvGF?qOF@>WC@R%|UTc`K+B$JoemifG)g5nF^}K4MEN?Tx+Xagg^A_53sGp}3qx10=TCufu$3FeMCG61gw(+

Oceo0vE5RMXrewdOA(UOR;x-=*aKE~3m@oT{I<(AhF`X079G;hx=I&+>K| zczbd5^77wP%jD^y<{q^;J(@1(e$%Rl#<^2!Ee}0@-smJtj9aQyJ&w1}{&ky5mbc5n z+nQxLd1G(XT5pVp`ni_cl1KB{dDAkmlSbBl-a9a+av^<;% zJ=9v=+9@ytwYG|JOBMQKwAP!YhkU%TTT7Ly-|@Dl^1{*C`)v+*8+J`%-q>d&>-G0r z@(DQJ-u}yyC$qfG1#fS@9Kf6QDJzq!R6Y2oovMm987;pSXP!`x|9-P%V|9;qyv_O8 z>5pZ3yAr(JJ>hEkp7Q>rW-TpFPZ64RsY7e|d9!4uhdQ*A7{7%UKW{PBGix1h6IXov z>@08dz}q#~HN#sbPmAAQwMD<5H_JH|vFjr0#`9+E%JKH{7QOpsdAka{O?@o^Z^nKK zzbZcpEb58vlFBy6lbdP_4Z_T0Atr z#UIa`rHaMFU2B?o=wV#bk~hs68F#!Lx9Kl$$?~=UyuEW{GrUFCE%Y8s)m!86X4z5b zC!PLMc{^%mmyc$7TL|7Belsy|+$kr?PbIbHVY~?4ws?z~Ksmfs&(z&-ckOd-$1HD) zz}t=IUn}3!nB8$L*`ub1TH}x&>QFz|=HrdKpl0UKt}K3fw7Oakk~b{_`3%v1@ z#XdvhTx-!0u|-77zUv&3>5Q&l@u{1LM@p%y?Bh-hOq$(ywHB zTMFL#U%WEkdur*8{doG+J}*u2&|0jkX+Lj8RJ2UM;tB1|&YLzXW8Cp}!8fn?P?oo4 z;O&{)o8gT+qBhoT;-r!Ne%@NE^CG4#dDE(BapN=`Z$BJ<`z2Z4mV>udeXh%QziB;a zJ!^3-Ptoz#<*m$VNZyK=ZFNVEw~aa-Bz`?)1$dh^H8F3g_EPkTSC%{$k7e5B z@K)`_@pj@v_w1Fu->wF4Bkyj3H_LYv+%^5wnul?V)^c*K<)r+)X;l~x)$EFE*}0~< zR`X~%%i*nB-SM{j0nh#+%iA^J?X-T^r+rVgX*62g&s!1QT9~oawdA44&l{bV>K>2d zZN;V+{khKD0~=QM`aqAp_nba-<>1-7RKKr(=F-HtX&*9An=Sr$!JHfq>)r6*WE7^i;LuZR+_CUoLptx6a$DQ*V&ZDesU>&K7yo z{4qIW*0LYwvGc}N%%*A0uf?@IjK{2%yfL#y8>>6s{-}z)p?tCSu?y@da!tm zeKs*1;}2b19B2W6C*cCMPDw#LvVS zlcPB~M$OUWB&H|EXiSVH=0wGUy}yltA9 z6K_)g(q8D7G&z(!QaqWrkglYPQe0C@+9}u4L+$d$RqRTN(?f0f4R0?T)B?C|`}y?z zGH+|2^WcrOVrWh&MtQ_ot@TgWczVM^KZmM{&&+i~CS@c$g%E-dplc}^AHcxOsFp@)h;NBH_# zSF={*cX{I~YN;YUlIB{;!?@%RiR*a_>7=mM@b;6%pTne_nu0<`A$R16FctSZ<>m} zyNm5-&r+>idE0c^iBCAZT~g-l%{waaCf%#tU2N^!gXEXu$-GIcq+M(Ll1Ji1Q;RvN zxi%l(3f0a1_S2V+eA40V(lT$)y;6ZUc1!I$8TzT2vsOBpH%(Vr?MRcy*F%3kycH&9 zcsu1gEeANfT~_98QQsBm-c#-<?ea2jo!_X!Tdk&~nWV-e^^HQhPSi+=jbCG-mWh5HgWJnjr*P^ z?7ix#>F>9YD%Ocq?Mm{8^pMP(lv#>P9Twt-w><`b!+lS$Df8Cjt}49w*7fx#^q=~4 zG&%LWX|m<%(M*`0WZu|0S23>ChPTNFT;cxrg=@>aZQNLeH)X978-YES^7z*E^~d8) znt&Ad?Ofw$EtfZbRg+EWF}(eI-vjXLk+%K!`s>QPJu-Bq{ymMAY_>m58|0?q~dQ%17G@NKW+%%rT7n=9}pox2ndnMSSF>Uq=bTI0!cr|c&kZ_FTdZh6f8cGO8mcQk;#U8u>uU1k!<+9;HGYkU6Ntx~)rsM4V^0_Nw#e z%{rZ`pC(_wbW-tnvnm?i?i#W9RGYW9;BC#=N0Z-E%}q{^q}dxijBB*-TAmu$y1enL zQYVs!aY;)#W5-?IIFk&k8{R(q=`T6=TRZT!;*l!6X?oLmqF+fN^BKiNwBho@|{d20{eCQeMrn^mi{Yxa|sW-aQF{4Q_I?5jyW*M@Y+ zoFRFVc?-#Gtu?%L|ML7PHg6rk+u$cE@WzQr+NxFJjedHhxUYx)5bg37k}YJdW-=l9 z^Wm+qix0nlJMI|#LayR}U+4(ly0m*d+xL`{$?6lzoLTAm;mvnPzEjPIx56$AZ#NI! z2mc4SB5$3*+lDDAd5it2l&0!iSMnE6LHa5gphxP8etNi;9^beoC-YOgyh+trg&F9l zrl+6{Z>^u)+4*^)GkAOUsg%4~wMy@4qr~g;ev?i-WU4N2QZ`nnmi$)S@OHv26Hd0j zf9nF?mUeg|-g|2GlL?b4=1{S2eexzvEM(Vu-lXc9c%ehX+ot>4F0^^;3f{)w+F-nC z_O9_X7H^szNuG4Pr6An!_Fp&d_G_EBZs4uQGgWv?c&{to8`7@pEgp|IX<|}5_GHrW zmZB@eTZcYh_?pdIckuQ>=T-XmGzIf@#U%TW$6Jb%N=q!o7JKD{%!DKU%1ZZtv7hvAo%^`+w=eb8X)Gg17g3JyoCYDfdgwJ^E_qNxt9c^sSPWU&E@@C*#SyWz}sV zkKyeX|My@Uo40=8?Ui|T%3FqWsQ4DU_ghFsiH(q)S*^{Lw+`QL{R5k~{^0F_S1a%q zGQE(sSxs1zCmnCx9kr%no*wq*^2V=b)m0&)wO*G2Z_-XP z+)E}5vmZTgTvaO_GT~(2LNZ&^FuaXCY(@*4w;RFR+JS45-&5cF*6Ja1PBi&7Gfc-D zr$NoGG}Y;kt<&))?Jl8P!`r7`*z=nmstwxXQ?>90dY0WgaHiloy>GG!OMB}k48s0uTHN z-iCs=w#zf%Ek$oNe~XK!kX7gPkVopl<;~)x;#o-964n~tc6o2$&u!j@fwvdm&VVa0@GhPT!ezHqk9+i>u{`Bte1 zKS_(aYs1^2(~h{==4}Lc8}eSA@)qecSh@qFPQFy-Bgb3c)7s%5KrR0L`jOzR?Z{`7 z-_zLVUaJ$?{&`G zH?Q2mThnCrG|beqCmnCTJ1X#Lcw6|zNzQvZ7QA(6`MmyqOTm2AJ1|C0&8*TfR;zBM zx>B4I$cMK=MZ?=2dklHleox1Nw^zq`@Wx#nCQ__K-!P57E$-J>TJI^Yv$*J+MA4wZJTV}IlPd*ai;+uV+8liyQitZTs(8S=}d^Pn=XMt2leemACW1v$Bc(p56-H z+CQ5yZ~WepyVk@t9^ahwQ|s?Hc16u>)EW={lGfy*N762DbV}8wIM-4a{N_FV*)R8N zYV$S&yj|b*CHFm*K4nr5jTqPS)`+`f&n|ELYF4_yjN$ETH=puzDFKI||94D{l{7c-S19w>jW#^vhLvlO_`*?mJcK zbl98ZN#>0i*p-xx9%|;%=#aHi{&c*h=)~M_U;pN(SJ}MX4&Hk9d8M)5Q)v#q@pQaN zXDU_q^)nB(l!tNYvk=akvJ=DG-P77%Z1Xl3ytP`;0K7?)(IB&}8hPQA2<8%FO z-sXX~W&K}mtoM|YRL-H|&p7V=7Ltu!#n61ws5Q01dDB!WcnohJeD~OcZQkaCx5;;U z@K(PwTjmdZ<^EBc%(_jyWgy?KD^&n_jUgM!kyr4$%YENN#Bu5aqd*(q2KDc zbk{KtHT~4gsnO|pV<%D-jYnD=5|`?_yh%Bm6y0=;0&&o^mqEQ_Fl}E^q9L+P6c?Z+QFe{*MnT-*2mT-T92& zo1A(2q#nz=&N#WSvxWD0;Krwqnk{SM8V_^E(8WWLGD{sw{*bs-i*ah}kV|+GegBUy{HToE$BMk2RpxER`yRYWbsI6B%v&SwNa{J6H;GayTSDCMw%^h( z9^mkHcA2*uN3GlOoQAWP;i;AGtAvyF@+9+?Rh2>>!`s}xz46b>Y`fpiDf8C)!5Z+E z)#-%f31=+Blk(z6QKEk#%6ertbR^9~Mg=azZ9XUux{J(W&@ael8T^+pf1mth^UJ*5eEpm5{T5PJGck=PqzBE~`r$41H1grCunWW6v-?fz#lyuG%32>yAQZ9ku0Smy1u$13oqdAl_p zX#%n1n%6f%ri*e>;JXhZ4cKgd#=YG4W%-iE_-pckpo;eGWdG9JFX4cos^ULlX;Ueuq$e2w&Lb~8~TaUu5x(0xXjza$u;0Dq329~ z29ajSc<~e%p=O7Ymi+wH9g;`N$+gsyUy8GCGH+4_DbCE)1;62~Gmm?}@vBlNzH#RC)vP70rAN{(Zz0(@PwK*5XLwtA_K5== z-YzZkHlzK9{NGb)GK^byb=7;%*YEPi>b_c2UE}estMR+MX;#rAb#BECZ%Zehw8`P^ zvNCV|rq?NN+;0qBJOyiFRjH2AN$i~AyeT_0yzSGY^+^tI&C0xOU6TQCA*UwY?V6m@ zTFrdi`;FN&8S2d=O)Q)@fj4$*<;?BHk$RBgzI_%? zLCPk@HC6aYq-IWPO-_x6wdhagjjJThYzf-%Hf!f;Z5`gOEc3STxeB}`C>l)(?iXDF-|RcG;ztp{M7FK z#;;1*7?(89UCJq~%_`2i$-L1?%}%%@YI+LV@ODg#7XR+>c6FJz!E-9`X5}f}X?eyI z_LWIL`P^@LPE4wm%$u|-McnZA=5N~I*VSzM@2A(4d26$_3U3ngQa|K`TJkgQTTA1o zhuXd0*oj6n8~sw89!X1U8TXxPGH+5gR-~5vg&i5*PCT#oIOl%5w#?gxp6|HtDW}2w z)RKqsJhkQ*d$a#+?>A-+*|l#TjX#;UkZ!rQ(23z~-HL<0>F{=4nYR^pRN;*|8%66+ zA!J9IDpLNCwNg$go{qN^9U9(Ry?5##9p0Lkd7J)9gYlMP4yEt+xYoVj_*Lob8ZoZt zEpP&cx6?cRWV*xKzskJz@B41@dm4MffqN{mr0J8jT;4PpVtCk9jQlQdz7;k8g2(W- z^6@=h0B>tP{QLDS@cSIzTTlTwO@SDr(d5jkw`_sSsw>R|d_693z8RE$1KbI>{{Gf7 za7Er)g16PLSKv*`&mUB%HGav%ICE0dd#< zKUwl~o44!1+kFEzeRLjF&aHR~nzQ4odeFW-yS#B1zS+3e(uTJOe*cqoHgB!K+tkH1 z=8ZjPxc92x*yVG-h3qI~EvK5y8=ZmL@OH>~J-=@A)*8I^d9#jq3*2v2#+W}J-dM3# zI-EBp`G&V2Et{~n&08Drw)w{QYWtqn_7s$p^dgx%lJZM&uFXn^^QP>?@HTtvE6)Aa z7QC%lR)sgs_m=cfYx?x{YjV=V{M7FK#;=kGDK1rEoSJ#4>5;UrpLz7Wg;Xh2G`t{ff$#WNtu3g@Ica+s+vdSa%ujft5z<5HphPNy3I?MX~y}LKL z0leKh^!?htrwRS1e#JN+Nr&?$Wv;h)I^I&8i{b68YyWeF{ry{e@HXh48uO+(9pAlc z{4Q^rRhn+=&6AF|6uU6I4Ow#KJe#);;H|}`D!lPc=ZP@R-WaE*hg##IKSZ;Z%NsLu zr;IZLwd7%3%1Mu;xt3b;GoBA`g{};5>qmV4Ae*<2;O*s`Hz&WR(p0%4NlR;`xRghV zGmoUD(Uw2Bp_4D@KUR7;98Cq0a7w9A_&8$Hws(=fd4^;mcOLayS^3!T8*Q}_cY{0Vz1?iP;2TIPeJ2sbWb>M%GsLxZNk>GPPTdL3f?+DlmTy2 zKT=#eL0`X=M-#7q-m;oTIB%MH7CeTxb-#Xhgw0zw@V0Ji72f#0BJ1*(<47L*seS!i zo0Zn{CUwg=bygipIXQW1sjkZ#Gix#^{f4(=e*LjO+q`uLZ?BBq67M}_pPYtfj~b7Z zQ?u6PP3o3asD1rh8$(O|xV&*b)KXVcX2uKJ@b=g@Py4*hTMzK|!Xp)UV;-poO`I=+ zCTEC;YpJC^lX+w3%%I6kzeY2sq@_HPpK+HrepS*bDjME4kJxg&&0A0Kwx-qA+UEV^nCG6DjcI-i&``Wzq0&goOR^ctAw|ZXd@+RGB zA-hO8ozkE0h2+<)P3A4bT^ru6IQ3r#+r0G#Z?m4rm^a`4Yjq+nZ<@1Xw;I2%N0VRj zFixG!n{S5+e#6_y1s^zn9?}QAjc(Us=YGvfpS5(SkkCUZGd;dca&1;xztei?He}cF zc(W#BcpETq`F-~N))%~Wo|+SHR&NzKl%^W8R?4a8P0FB&$LL&Bm!5dMu}VQ3-VVP0 ziSupV`hmCgPvyj$)teV@G1lh8Ta3v;>@ zoq131>Gluj|Nk}+yiI(j3U8Vzu}_WG4~Aa zN9LE}^hmmR3d}$)t%@CI57Z%blX(lNVy!j2z13w1e!*q&?=Rd0-k!g`0&o0ugzU7Q z*ShyxJ$D<@TRh&Zt_*L7w7B;&`+ge=-tK*=0&iBH+`dnX!8vnMf0TI(IbY7u%42w& zxuElGo3~-$ZF29{`u8-UR&GpMdrVz-2XC2A!0`6v6?c8X=5085>o>m&Z`K=KI$tRs z!h^&!<5FG5>1RKTyVGU{YG$Z6%|5BwXK|v3VN--d^h0M*p6YbB*?$YMv(;K6z<( zoD4nE?j%3s)S8^El@D))4h?UA=zjXyHg6-r+sZ{Aym1%Ybyk{FrPk#4^@sB&Rg~hI zUBvLD-a5Q- z+|@R3qruxPgWBrfQ{Nuy*b_;mxwlpW+VoWt8XZ`RN z>GVr~l0ed;?%MG7`QJX+%;s$jc-#0^1>U4{l;WJ2#zVhG$H+t7Dv z%v(bLr8!vp$|RnAzs0CntDG)xkvcKF-8lOzt8LyUfVUfl-;n&Cl5=Wu!Z>xvzOu?A z^^h%Z?3P-pE_pO@Ur)BYN!1HGHN4$@`R%{4d7B8{Hs4!?H)&$^9Oq|2>a%zX8mE*& z6VEDtNSL#^kH6(!Bg^k_Pf)-n%0)RKpBl2+8rzz(^J z9!Xn%!`rd{=;qvSQ^DI!TdMHJob{$TD{4)j+43e;^zBN@6B2iM3(1zS*6_CHfPcTl zzTc*Sx7K4ixbJB_OetYsnY_8_cx%MT1U zvek1uSKdB3Z1}r2Z?}TC`&xE%@3&e_kKZT6?xA=JtVqr3zHzS2Q%ilO<4wxO9nn+J z<~==Y%kbtlZ!^H#yzw5qF+b~SG}lsdt#p^tPtCQGmi)>08&^?FRT%g6bFDN1dVICZ zn{S8oQ(K)F-g(e%GGr`;B$2@q8F%7HdOujeM^qI_?CPR$QrJN!CxV%YK0^^3a zwgbK~)8_3q@YcIcr}}wM1NU2ERdYHnZ<W`Va& zleR-jDxE1k z8l6=hO|4vcTYSfe@7VX-9Pqa2hR*JLDoummD@s~3fe;VthUC#Rrm4b-QFE;nr-wS* zosv3M3DH)E=6>6&+2NPlyxk7o#!suj8+-7b64%Do)_yC#R`gJ(7jNf&#!JzI+?c&cWQV$=!^g6 z{C>+jz+0Yg$rYCmitVJ#LLyyaw)UA|FinB^V z8{Q`G^57twxB1|0)w3D!X4S382KPvj8kh)$K}m8gQiL$r{V3p=Uo1p&D%ooHf(mrylHxqJTc}V z)v8b4q;5mvE^i^(64n~tF8{-Bf3kU71m13VF=O5|%xF9b`>Oa}`}%9kTb>hedCRK0 zmB;Y*%8K_o+q~Tg-d^s}E!}&n{2sEz2K^dMek6a0Cz&@^WEDwEe#WVphjB?ue#Ry3 z@+MVboY@$cbir?UYdPhZw{6}Qlef7kd1F5zI-&m(tIQwbXP)9IFdM&GE6skWHT}>} z?eZpN<0{F+IQ<1}cw4{Q@y>rgT>{=#zwE)Ab$^z4km6Ernm9ew)*eee(4*1G_nRhz z(kwRPc{6oZX@bs-h3zE>(6sy zdfo!NGQ554@j=e-4_OA@rY)$zTVUPPcMj{uKEY((Vs{dIckDk`-VXofXFprM-&XIs z^BJFLa^~ridMxic#EZXv@CS?eAGBO0Amb=Jx%r<5mG+%^vXuT#soeXPja zS!Lcf_wV68r<_I%&B;*5s3qkro`O`JacU`BjCfXc^Wm+~t$9wb9N6?|hqtrKysf%3 zC2v_xBda{VCtf@Su8Pq~jGXM5->_4&uFD&p)Xc2WtWxkB-tPO)E01@0JEzRsg7qnR zOVLl&Z|ki7tNJxPNFJ9rI%9OgoYbtV(Y|Y4-lQtNnVF}s*6_A=#Sh0jyfrQJHtfcp z`M;;^vsSuz3Lz6_=bAhk4|9fS*3$FFip-#i7d(cy7hk+~iNo8uW!_pXtH7I7i*JW=Hh+_QP7#>3CyxW}~L3pbc;RFB`Yq;qAOK zZ(BB0;Eg#OMf0RIJr_?wvnVRc<-q+B~Aan9^YEo^2Tm`JKrjv zzg<%1?Wy-Yc=O$<#&7jp@$J*(@%8BMw_0`RyCcaTk2kBL;jP!bSG09_yR^*P!cl#) zeNTBCXtdRHiMOoQO8ND?Nu5ZUB|qc7Yb8J9Qht{=IyIVAsJXTfH@xlj)t~>t;q9_A zZ}T3^m^bz#>3X;^^t?&Ch!Kx58JD*hofmQ%-fsTcy+m&VB zwp`!Oz2DfauTIft>1PDrS|R@8DKtvAn%hR2MmpZ4YzcA0+nZbd@PYH5URCDpg$XHn z(@cP#gg#UA#{ATpTJHU(Sw#;kYBbkUOKa%~(JpVS7?RmqYk2$FhBMkZyj@-9?eWJe z@YX1wdTYO4+=lZOQa#VLE^m>l8{TeyVZ}ZUZ`YK0o7c9#{ymMsVeFhz{&>7e6(gN; z)lXTjy!~_SUk`P7ySB{R@F_Lst+uC7tLj-XsmY_~O_MEQV&;D9bL`*TzrS!@nYXr2 zRp3qYzDph{&OS9B`a`r-%e~(urzTF1Msr7!pB`$BpX4O-#%!!YO;3V0yp5RC^SjRd z*1XJH%MJtd?EEy3ID&I6MlRMyIBdYU|}ORZU} z@g(!6$K~M_E;Xku>EbDH6(`BK zlp)VJ`=QqKkS%Ws6%B7cowx7qHgD~~ThCrMy6-6`pOC-AQdWN7JeuAlk9)u6Sv}8M zx$<`4QPa+`dAk9;ZM>rbZ_*iQ;?g9gct}pJ_08|{#%#6HzPr%$nJaHEE?<=B+(= zd-#=%dDBdw9z6BQTa3x%%G*E3op7+tTL5H7hbsO^-%%Ew!{(@<{Py z-ndFSQ+lMX3UR~RqhH@}y3JcB@Yef{l)NR>E&Uv!@o+EH(pq1SKX9%!i-N_|32U+nsf$l9R?1`|2^d%Yo%S@YSl^Xx_aJ1I@$J~ZtVB1 z&)N4|7x1=z@ebaahO;N#41p);^-@T+((#sJN9H}9`R$ckY~H$pw};pN=<8hnC0i_NNVR+;8*N4ZO`8e3Sk?)tsr-Lunc@ z;vxN{<4vk4#bfN+H)pQA4gBSlkv4DL!Q1e=Jb3fnX|4P`eQNfTjyGm#6rC$?%UZTu zX!F(sytRKj1Ky-F)tm^|N-u&YkFTe3dGno8I^LuzJTK0?;5YASzejc%YV+0;ysaNP zRR5kbf1~Th<;{2U>3H+4Qt%tzP8hi7bep$c;O(({cJP+L&!;I)$Ll*==~mIy)$^vQ zlHf7C^=fvM^Z$SA4c_K%s=ym_O7|*vDtUb4QcmXa)yzp`+n;K-e!&%mhL?zlYxDDebPf}vFG9`gk%d@%TAa_iqj)$ zX|2nflp!QusBUzMiX*}U}yZ~Y(e;7!_5NT0FSNH}0ApQqvQn^Wm+~q2cZH zY5VuHdFuz>KG^KRTga~KZEbz>#wlgRUaq`7*Z$*6ZQlBWw`WHWPxqd3GL5L~hc`|y zE5?{JSKc~L+wHG5Zv(*FvWIr?mhsQ0F_Eu*RQ!zeyc3b4Z7p{8BBh)%2j}jhR_pqYG=z_iqE1to@eF+aU1PZ(If5 zH2E9dlgwMa?Jk4_J#T@#HoR@_@P_mMN4gQbZF!^$Z`>($U~jqo8Hm(pZF%D)sikhE zxMpoU-mL0|w+oJFb%cGt4F+$kTa9qvQ)zeGQ@pmk`A*V0S+7s4%Ueh`sq;ck!`o$R zzS7?2Z3uW%;(q6b< zU#-bWPcm=5)#+yy#;G+P`a`tKn{PH{w}!XY(^egA^EM2;ZJbgA-hB5P!yk{g7~O{C z4ChT*-SGCA`*-bX^EMp3J^54x-mH7F^xdfxm-zRMEB96EC&cgaChd+@=#h3O`5Dhs zyS#B1?1XVi({E|RTerPOPqTTu8N4m)Fe=@9%6?ckBrdIGero!uH6*0tO;ep7YIenL z>4~9T-dKg5Q2Tlc{tw^NGk*OUo3|0*ZSJipdDHa63DfVZIR{B=a&m3)6xboZ>YGio zR^!ps)p%GdnK$2T%xtaw@P0e>4EzHO#ore_61+`%CIjBAx)t9$o~*_n+vDDEu`7n; z51C##Z_3UMZ^Iwmce#DPjRJ4IyNq_}C!?n~d zZ~UsH*`efNyx=$Y+yDISofqu;?H2HM{ROB>#v{L9%BY~IFzxA(i>;=ZR=o~rL)hCNrT#r)L1ebQe% z1!kkB(^qTq&?D)Py3Co(8>?^?H9ZO1@HTb&`ffIFW5L_{+cV%Tp@)k6#PIV*Vt&S{ zlkYcXXmqXTEpVrXw|=MYx7g-w9C&;9rHpywnM(a@Jkr{%;x2FX&{bBuW1f6@8#rr` z&D(hJHoNzjbnmI}>1g~K4<{L-UEV^n#aNrwG;-zbsfYJ)ejnEa@HTjU1>U$*>KIdH z&OB??KX05&h<164G-vak{^RhkcCqibiQui}YZZ9o?tFXm^|Mxm z37r_;mf!vRAvSN5z}qwZ#-@8urQOAdC+sox&m;B2n>4YkcJ1;eRSb+9-u}|=*j6@g zlfm24MOApqYS)2#NzIj1SI=9cPr&7kJEE3mrHmWiPB?$-;Wlqmz}vWWRd`d@%AAK3 zfBsYIL9V~H`}_zvFO z2?U;n;ceBljn1FHO#^S6mSoJE@445??@n9lB=8hceh(OCc)VGYF}(fZ{9XQR^L8tETR3EV@_Sm(leG5h z#ecTEg>({oZ8C4t`AYGGZVhh__dVp}Hg7Y)+o-|q-l@Wy^s}CCT(d9cq~==2sp*k4{l!yYM6*icX|DJ;IHWvT z#knt+H-43xol`5jGQ2(dSj)~fZ?}QB4#Ow7?`e!r8GB}aYIq&I-U{6s-d45V zGRx*|Hh8=5y&CYQnPiO9;oFfvZ<<}l@Z`$dzi%Gn{Qt@4fVUYVC%W&cv}>N1h6l;d zxTcomk>XNstXo^&*om|YjYnF`xSlswlxE1xmNxg>;!aP!Y~OFUgSX);Ja}V%_D1dN zp+76lek5Hy1?j84oiLAPt;R2TG$e4X%NxH+&CY$}g|&vaAD#b(^Yg-7@HSveO5WI; zW&-rrgVx_~wc1@)bzR;<=AY1&;caWjAKYr+Z+C#VEw@b4zo#L6*4tV=Z}rx>Z$H`c zmQdaBc2d7izp;6n2i{&-S%Ej+rPK+vs{W~F!snvu+K{@^+I)BmnI6|tOFLqm)6SK* zolZPvZ=1LI;O&8yllAu-cOhwM*Njs$596_QJl<-xBWYr}@^<&S^*^zBTL9jsj`!eA z`jkm=X#%x65%yU;1xBc)&gr3+{ETzg8qKx7n)zMc*tw*oip*?z%zOGjZx0@3^R^JY z-T0UXZ&v=Q?_ff2RoBL-#eUM=Z_E&*L*JYlzss9um9M8T8N=JAvA67J^R@`QZD~D4 z|DI}ksJ9c5`f+)qlUlRudiLbQTVVo*x1lSRJYn;8CwN;ksRD0}GVPE)UEb=MnULOc zid1B@05J~zVoT={pLG8Uq7ek@+M`IW=KCXTiWop;m$RO+3)F6@YZWu zO5RwDeNxjyt?|%LtyxPCHS^S#H+JHyr5#CmT;3#SVBGMwvHi?3HgC(o+s4%%ypeP2 zkUp75qkU^JC)XzPCaq!yU+tTderncbT+%LYl2eMaA~pSmxZ!R0FJ1A!W!_fry7L)( zG&%G1Nj;W#opEyE&m`7$oR;o6Wm#(94O)c+P6v}kHKx( z)`PxR#_eN8-p(rX_TY?+d6W9jDxP8v75}JD&zq)mc15jOOHasJ=All<8#7QRbZB_% zeaLOcJG`A;=Iz#J>xj36y;Nj|bMV!A-tz3qcNZGJ%Uh%!8Qz{c_Q}=`Z|9VG8_;#S z`<`0Qa>cV|&-7Ds8d6;H&`(W2wd9fd)bl1)kt#}Y=AriWbFD_Zyv3+k$oU~}SAFeW zhqtC>-n!1Nz?)_l_2wa0n%>g!#>~DIvs%kM8tw9?S(UW3OJsMre zV|Y8SBryi*+B zE-v%-&TA=oi!p8AoSM(31TH?l<2mtQew0>gsuuI#gCSyzTSE#qNKvZ&v25_l>u?@2Mug#v}D#5AkH) z8hv-1a5!(wmM|H^+Yvt;^15@sU0&wx-K7x5IN`rT~owR+w{Iw?%d z@YZq7&>J1zt|;@iYC{Fy0_#?tu@rXPB!rO9hN(pulR%bS#eT~SN^dW)yyO|uJY8iu#s_ITBOPp>ZX*5Tch zylMK8JXW7Id4Ki1NxP8Z((a_VlqV$4TH(AgTS8Zcx6kiC6Ti-3+uvWfrp()`!)Lqi zsdZ0!@gS|$#JLyx8P{m8rRLh=DR7c>YBaM^b1mbn#ki!Ihg$MWahEqKTS&aH*6`MH z_0PZUyrU628JBvC?O|PNOi1LfCe;mz``!Q{wc_y>(p6S#IZ2ndtg2Xf3~#-T?YzO^t$CTZMWg1p z@2Qoi^q%@M&s|G78TZZ4wUU-{rsIt}qL!-HD$ZSJ%NsLWyED8UbHV+0IlTR=%-e(q zJ$SS7mpE+1xXYV{AB`u*G(zfT%Ui2pF@wW6@UxbUP40^ z0nM`(dvk%~S3`7+9fjn~6}bKW_tA-E;EKGp1aBM1+^&C6Lrx*~THh0i$6M^3`sR0e z^PQElqPgHUwb|t>HgDI1x5poD0N!HEX9sVYPr&fD&)+`N)#j}gc)R2Jxs7|jNgVQ< zT1lsyHY+kSZve*Wr-zz;#w9J~WFBhwev>jY&MM5tIQ<1}czfgPt!CT2wFYlvCe(m8 z_RpTBI6ab1m_x5)r&ZECHHk54t>iDBg7j5p z(^U8M&>y0uKAAI|H)%?&QqbmpYjS@3+wA+T9e8_ovIlRP-B~@9?xK)5>ryi(+d&aQ%=6(u0z(cbKg9A-ndh0R!`7|w?k%~Z2f-nJ(}DA-tK$SgSUiQ6(=nDeJ3X6 z35mPBrI>KVT{91LNVg$tlX+u@8lBLI;jPs(eOlP}TYK;}r~N$lJ!M@fzsAG0Suw`C z8qFT)FP;J;bW+npO%JuSQ?8}vT55Wz>Cxz9-ZU9BoTB#Ws!CxRhPOX|ZdN;+w~pX#_|sK*i#=5- zzlIsEmHbjXnKw;ldg?(Z@08SqbSqUUZJk+xH|D3d z@|V6_CiC=DOKTaYW=<*2{F3Hc>f+?NieJ^}khRP&Hg8?P+lJ>V@Wxu(t9C!>+xJdMU8J2F-i~i}F#ZBV@&6yv6}+wLwjlXE!DD#a>40ynw|VOU z-p2G=nEak{r`wZudE+_8*lFziQqL}LF*+&aG`#IS<)R`*-Q)u3=S?N1Zew7&{O+R%=PE9S}{4Q@&6~?(ENwbrJHoV<;(D$u>Z+DL- zy}(K0GIHv@O)yOw@x=JeGPtIY56#;spTOaWD`oaplN%`v` zE=}0CH}`(aDzk4VvHdP@kt!PA4(WRo{=#$d&)@ojw+G*d$Qw_PI?{ei--&7RyS!;u zg?MT;fn0ez|MI@yx9_)p;BDr>JKguR9;T`}9gRoYOa8n`oy3m2yh(FooLbXi!Ef%j z6ITz#FDNhGZ~ehr|0NB~n=}W_tn%kg)0MQNkhsfRNVbHvhPSyVTz|cNzYPFyo8Qcs zw}gHwzIVQ-;Om$Aae2!svs5uge&(cR59xT5vN5xz4R6<;+wbo-Zv(;GlY|eiqlqW;mSHD`x9=Qr{&hBQgTULoyHfI|*=ap{((zW$ zJCb@!=1p3aB5rtlpX0U`=0X8 z@yI#jlBQoX8Rp?yYI=ONl&5$K%qBS*rv(Sk-mE;O@2NEWNNZ!qeS2%HJ1t~-$-Mc_Euj;`ThrsFA7}G+6L?#< z$%8l6VxN+xzut856r|1>uT|$+^^nY4hPyMo_5NV)3pQ^pS2 zO9h}P5!*&TaxdDCR3 zhuu=sQ>bWoJO1fTd)xQh2=KP`p_II_E_auw*7P8GT;5`IE>(|FOUmi;COJdmg`FDS z4jg)m^ZN!zg12W{+!cPm)x%VMXO$0c?5b8%%7?eYP7QBM_u6-~eZP$YZ%f8)C%pNd zX8yd9J&pF=k(4u6-deu-2>wB%;{P9MGAVJFVwzjp$G2EyEod-i~|s>V@|GHWs`+@E*{u}!_0TVA){=HXkEFwSlQJ`2=+N-i`4<-tvw0f_-tKO9cldiMO+%Vw zJ;aNrP!FBN?jf8vWrv2hfBWepzqffC58kFu&44#$KNauETCEM|E#y-cvetKEx$?GT z#207SyiEXa{a4q3H_f!^q4u4&L-eyg`>Nc~7t((@)|W}H>13m(H;lQnltw|Sci z-UiO9z?<|6ux_L4Y$ORenI@b-&cPj&u%=V{<=%kvd@Q|6C)r^W6e zTi#-K88;>x(RXhboxJrt%B0Z93PHOrYr>2LR9*s`sP0B3AS<&(s-d-Ko{7IX)8Q|@%mppi5 ztwz_yQy@RSI|}@1uJoz&t?TPg=FK--!o&=3qweYC{C(${;B9>G`_sLrzJ12<^YcZm zsZ~4$jnmg7od`Rmmi(H!T+5u)>3CyiO(zA9;q8UrwD^&IzugAj`p(aZw-o(negm+F zaNd|rc_NwbNK-c*Z<@@O$MClK=O_Hy=4}>u+x)5rZ~WA-mK3KaM2F<(TIz7#LUzQp z?27TMbUNNre3}ez%~t=9^ZW2;gSWN)9?1Vat%vV?^5M<*)O`IKCUfQO`CqTt$-dv_ zfVaCA)qpo?N-?I*TFjq6Z>-L)V(VnyGVIFSZ@V77*!lMXZwGJV)@8sOUj%Atk{RwX z^>;+!yk)hEtnzT$%$dv^R|RUr+o^y1)dBYXHW$407_=h)_cU-{sTpJ4aNd}$R+?N% zJ*4AJli8|hcS+x9wn&d}wJUu{G!om#HEUD<5&SvGI;!P~qc52kxhHQ%R6r^dA! zE#(P`7f*rJL$WaswI(P1)S9&#Pcm<;NUh0idCdLx{~kE|FE(!rz}vLtDS5MMmEOCo z;vswIT59%|?foX*EvzVM-?hH}e0iJr->2BTEd*~jzLSzSY1dL*ns%+?dfw_`Mfmc5sfH}0qr zb-c;@k{>cXO|4vc`~Ko@zHZ-dOTgR2kt>tmQ+~fqt(g-2)S9)v9_FWJ9>$Y-qm!E1 ze6=Qz#E-OAinF&|dE4{4js_qe(jcr z-RIOwQ^G-NEq0tAL8+GHVVpW1Zc>N7@sLyCT0L%D#cnw_DPHgx+`jkq4P?e>O>Zu5z8N%C=!u=5wWwX**aBYUOu%<4&o4I}BON zT3O}EmA7M`*l&=-+xcbQ?p)OXyk#|co~Fwizq&o?TzNbFq88tFc)Os?+t?c(OMXvd zoOL~%=Jv;%bW*wUwrc6a_`mmV`}4O8%e-};R)seW8`4fSYis36=FN9UzWzLSohxq# zbUAUpbH80w=I!m(8So}eul`R#>bxG}Sxqlj-sWBYx3?VLE-v%7s^jCy?`f?jANx-a zil-2}iZs1E>*{&abVZM)&CjR9m)*0>;q8(#Z_8(R@RrplPCA9yarR$41*syP^hj}f zveKGbl814ZH-42muye!P)FV#c$KmbLGH+v^_25nFhn#CX^h;XG!#HO}O^=j6`F`Un zW{a&gb?Kp&CgAcW?Lx}HD$?45$ME)UyB0rnc)P63+rX|*)aQFD?JFeCd8WJHLaJ-l zvO|rZ9%-$fH)czr4R4=#cfuxzw`OJDw$93!H+~i*onkLl|9qIWr2O<0PeD2(DMKU1 z!`-CUqsnd8YNCr9NHW zB6Vwc`^vDB4s_nrE6cnso*R)jX#$bz>Pl0WolAbnlPzyjl|omBw{aVq zJm>IsRhhS2Uam233HNhVemMDrIh69SRy^KVg__;Q)~uT?ZwZ|n-tIl=XYV+?U0vp_ zcb_Noe@_#7F0m2Y6OXspo%`m`mbZk@wc+iq+xPF}@ODj^xA*6#{d>yq3PM<79%|;4;u;UvN;=)NrEF4H^h+MbL$p-O zz2E5MXOMAfsVmE4c$;$Uk@FnhnwNQ-ac2eI*n?E79^xT=rsGYsYpE+KuF0wKNcmmf zG}%&k3~!&B@#mu)-u_kQt^fL*c$20f#Z&Cp>s@wvV<*%xI??2pJW^chGgsaY?Q+px z;BC!^f4{y3exJj;gI4PwRNt*iBdCYC3tT;P6{ClEz*#()3vQo7JN{c4xFT;Y!P}Ed zJ$U0ZtXf`|3-e^9efvq~%{N?E9@1cpG=q8vT1(57W!4hvF$ne~c*I z8&W*0uCmI*K6B;mkiQ;b{rmNwY|;GdH|#p0?*Y`#o(7-X0zP^hfzU4LQfG)@n{% zP^4gZ1dI*yxo3ZMBb$Dx}|uEJ*NIyBmes?t6lrD=kn&8L3u8Q zw-*l_yTIn{2JklW{fv3zUZ|y>H6E_@^>ZzCw!E<`YF5ljGe0%+Gw$+6XNKDFcK+rL zf3kUN58m30dS=Iank{=7o?z0dFu$?R&4R$P5KT*ic1{&`lUQlT$4YXH%%3GNIx}CoN-^x zJgl3{8>?`Y6sO11hPU4T`#OH1Ly@;m;BE1kXWjQy>fFko$!#y?lzNcjQXVND&YP6E z-r~u;v0G`jQk-iG+VFPTfpd1Zzkllt-fnrg0&iBHs9e>m)(+k>pMc@*q?L>N+PrlE zZ{4qd&b{Ai#Q^InKU1kc3&m;2uEHHr=UG>Ozp=v<+VD2)*5@9wdFu+^Hjek;O*18p zC&gY;b0yWv{L@|PFvbKza^}k0e!KTM)#j}mczg7*l)Ul#NlEjp>0vyE*3`=9ehc9w ztI34(rrd?O-+nUp`D1L}x`Vg*ZJw{s_cW_%OL^kmZ&JmOI8U5flQSJ}zEujH7~Y=y z{=xW#T*aSHdw{nQlXvjeRL`E~=F~6{;_>al<&7DreY@h?f;PN$f9hMmx9_)};H~GA zJ9zWp&G$s=!SC`GsVl?Vt25TEuzBkR-rm3Ag?!%A6!&xKJMb9u(cft?s^^(ASKj{l z^nY9bzVj!W^agLwOv{P4tR_ruC9SEoeelK!NW0FJx4o{ou%msy^#N~-*EAS!(o}up z;k@yzseL=q_%-<@Pp-VJ{l^mL?`QP|Z(};Wn9qBvnX0cxny@BMIB%LN_23ETO*v=t z{o4VzUio$Ve(ML`y3UNq8-IRb&y2@75!RCOC-cS*B`sx^sxuEgj58%x4t>KHkmiy%n5$O z+u@&>eWK0V0Pyx~*R}fhl)JFluK2U0#*@&0)tn*u)A1HkQQD~{r{t01td%QopL=)W zNj7f-!P|=29=vfEWRG!;4p|$LpLL6;z=$R@Jt1qECq!#{$d$KU_daU4&D$XGHs!^X zyoF4u9@d8Q#$9O6oqmnZDrc^|{rD@F9b)r#BY5lE^CkUzs+m%wc#5YG(rs2eg!G_U zn=5a>{^Wnmvw0f~-nPumm^Xe$L9Lk#{nWl|NrIHe<&92iP4%oizIC}%sfT#HS>2kS z7tU*b-U&8uL%`c>FMIH&*`2g&DK6!d;=Xz4XHJ(lI%8{g61!GB-q?9zM~1gqKe^~N zo41?5+r52W&h|ZxG07M?Kgzttm}h}K!(0DT1|DzoHWa)~UQmIzNc~j&^9_DR^3?hB zrkSe7<2$8%?ze5c-81mIU2NWlfww-dSKy61vfk*_%rFl%YiV@ITISdDMh2SqYgU>7 zH8W?WLw3rw>3CxXYH4=`kGbELefG6|ZQh21w+;hd$@V>!`tgl(kJOqx8o!=5siLNe zwAMGyTGY(LI6otL-k5=zC2jc)Z-1EC`Ik0tH-ooLcUIty-Ac6@Js!?mJxoAzQhMG3 zcVu`wq}`!=+q{hcZ%?hSz?*Mfi}TWp!#K4jzvP$Vnw<0`^TsOF%%IUBYc=^bo?LnR z%|YkkAS$T@5!0OZ?ReUE9Qj2S){`29juxoR_ zjXJ3JSvGH@z}vKCRe0l$LNrf-nmI$_T&vOU{l-=7%2zWFwQn9@KWkCDyz#5lffWsJ zC#+aC)8=h7cpJYlC*GvJaK9QY<DxcRZ}HottLM`5+UqC(k^e3lNG7yXWY_; zw@)wVhJWC>`1hU1g17hHt-zbKJ1K6}E&Y7TID4k1N79m?aY;)#i>JUHF@v;9iZhSo zVO**u#kp3}(%NL+q-=~!ndvEL!`q)0Ug^B2Yoa2AHQQ~G`D`}TV}9=zRqzXxv;=ZsTp_Qn3Gee-B)(UXohc16w1zM8w{ zTIQEL^h-Kh-lQspxZ!Q^+n<frY~Ch0e*M_&xZ2a

~wRyW0yuI^yO5QjD>KN0Gk+XOTA)RNnR>~vAeW#JkTUJ#H zU77oBL-)TrzfW}rczdMno5}B~@9rAKpM1YH$}Tj0CiAArp!68t?)cibF0k*nnc(e? zDLZ)c_%X5;s@^;-JtLNLi z%>r+oI=r>xJ=L=po0Ay%bLH*yAD+3B&D(77wtjjQ-h3zE>*t-5r`9v3>8hT0ohxq} z=KkkyHg9vl+sdaa@WwyAp_V4E@o+8ulBQqMTq}7Pr$3xGW}{}sdeAO!F}f|}G(Ru= z;ct)qz~=3C@V2b;hWdX`HMfVx!_yDvjoD&r-`%kvmp6VjLv48LbJJZvw0WBg-X`3Z zlD8P2k&vls=Ek1Gd1E$Km*U(VJ&a3v=$ACtQoFn{1D%>q=x3gSHoQG_+r4ktyxjrb z20!n?o74|`XcQgJTVw5%)J5vt@YZDY&d$FNI1jwF>Atak-qSpJF3vyCijhva>YwA0 zw5Xpl!`n-nTJCJ$Z}Y+1D|70Iw>(cI>g203kpE6eoo{vJr_%8D^=r;J+~#cocw4bH z1KuvjZ$(q~R`EonwNl)7ZuA%D&(8VP7&^u-G<}k^sGl;!Tg!o4R@%HR1aDJ%y3CyxYN?Zg$ME*EUr+s_eZMURZ?E-zC;2_~<;~YG^&dN4JcZa*V&~UPI~{K+Iyb!i zyz6hSuz6bo-c~HE!dr^ErMpfu4d47(<<#@$TT$cJczh=#`E%v%gFTM8&*p6@c$@x4 zMBbzc)I(g)TS&KgawYYdD{r$N-_!a1AmZ`<_a>mg4;0fLiir71#47F(bu& zJB*zt9d9YRGVf`#UG_b+e7~*Ub>}la)#S|6C-qp~b;ikscW>+BD!6fYDOxB!trAL7 zT&gAc*{4P)<0e&SX5UVH{j8i9>Q|4{=Xw9MOxw=3|*ol;x* zOLtdDT$5kp@$I2_3cjl}o#g4!^zZVfS(UtXM??wYqRpOZZdClYP6<9=F#{i5A#q5(*dC*^l}qf?_$>-d4GwSTmw8*i zDFfaTddTF>%_=`nCz-d9imVcnCuA*qNal@BsiMYHSZjFu&2hIr?C^F$nYRZ=yr2A@ zNmQa(KJ2%-hTd8jQD)nQ*P1Hxe#sW~Qc} zacYex#Gemug$@mGO&4{%#o_IuGH(MuXaL@PPlTr_y!q}T!Ebo` z@oV3E$l>kcGH)M@-t4}o2|262Qz4V_T^rJ8IB&k)(jU?}^JsK3Z<-8BkKygxno?9EKjoHIvc!nP1AO z$s?_Gd1E$7GaEgcZZ#g}_tor&+U1R3^{qlbcT~t}c(^Z4d>d1JQiNxQs7+MVHTWtUg?b$GkH%-j2|wlwa08Zx)W1>;k2}0wQReOGi8bI&nwS(%*kkEu7Re*!Y+T;BQ)zefkhJ7oYqBx3r44WU?cMx4 z4sTbMd0Y5I4S2KqOwEjM9{Q6fPiLd+biAct&+vB6%U^bXKE0~U+o*P1SC@GkHnjq8DcG)h zM}_Pw##;UT#%WN;=qg4|mp7@3@9r$W;ce_rd$e=j(`(ASwOL((H>scPDX!6?bI_p7m*VLNc;_+rx zH}~7dZ@qAo!`pRb-tN8CgEuR`7bmq^E1gbb@fI?@bi7G(OX$S#c1`#HhreFG?e7CN zFY`9_nVfjj^sn(G?AMF`9lT{e0mIv%uU-6>bHDwo%v=91E%omy&pJ;wV${`7yH-2p zu0wRX`;8Tq6EM8p{`jD?!P}Y-|9*W7{62@xx1|J*4K<>6fm4pPbdCD@Yc-{C;FO&k z;NCjqO6&KjeyT}J@b>ic4a{4}iHGy1JcSaUwesi6+i3&tvHtz6Pc^w7ye;m2eQh7q z5N1M7(B&;8TUKjBrXj7(mAAJRwYsYOpcdcLR^V;S?J0TVgsnelE`6e6v^PaXm_?^WzZ>_=G(3djcjXgx#ONn9D zlC-7=mp3V!Z(OSG8~3d%`CZ;5XJFj$cIMHGUa@&=1KwKoZk6sm4eZT}pBT0Dyh)tI z=qg6eTzOkLczh3=x3=Kz<#{Q2<0(+bm{^RQdfsAG&ogJPyj}mJsi)h#wF7VWznYS_ z7!%`n9n?}SjYnFW%$t;16Q@VgQdjiT!#FiPl6HAx21!fV80XqT+`OlEbX(BX=IsXX zwzOYs{d?LR|EZPQsvGsK%&Jy0Z=9;6v%1x}BTkiCs>QWZpDu6AOf6+l#tm;P+8z4e zHgE00+uTJJc;hapm31@cEk+OFywz$NzI9#Rd^1RXo=f2qVR*Y>%wfmdymbI?qu13D zZ|q;%BjfZ-nttlyDfq6UpV_21J+;#5c$02aX%~eF7~U52JLd$Ow~pX#z@Rqz_f)DC zBksFj-yZ7oev7f|biDb_bKCv)r*>c7Z1dI$ylq)hN4#mKt?_7PmCRdKRr37z2f1tZ zpAT<^>gIlX;{BP<&kLQw+nTrPh&O3snm9cgoy?mi1J6|BuazerZ`K41Z*zvdewTf} zbpdaShP18E_cZqDg!2}=L*M+?>6dPL=N$nS05dVsf8_g3JIf0{%c(hu`cYjXPfH91|zq)&Q` z>Y5HU9`+{n@AAe~nkw{2YYTD1+qq-kJ=*52CwRO2y$ZZBe+(^6CPpnOr^}n<%qkvZ z*P5IzZ&~fg%42wY;2YN*Ve{4tyiFQ;Lv7zveqN}py=4BqU;ezs-jS3)nKuow(z#e` z4R7z9Fz^nWx8C5b--;UZ#y=5d|B{xb>Kk`?V@2O=_3TgPEyKNTzR|m_iy*KdFu<_*4)xw|DHsN}Wqp7%ymZzx}q=J40>W`hm9khV;`|yoG$CbLH)! zs}DQh=4~K&>+{$S-ZK90r@n0X`Zb?2=FFG3W^X%x9x@2Lz1O;<{yk0crkCzCuBFyY zPx3Giwd7~KcnVxaO=pN^PU;vrnV&kDH#!5mGVkeo=P%pIzTa*HZ|f&{@D|cfy{#>t z0`Gy?9mcLDuqiE6!PpUgqe`F$-MbyPVgJv)-O2I`TKx3 zfw#faa^fwa&zdk(cS2oP2J( z(Dakcn`hl813gOL<)0BquY|LrssRKfK?*ckN*|ZzI6l zh^}4oc~6x+NBz{Zhh*O9w5A%hL+LvcsUOC@!3T=4%{QcivZSyt?ytR1I zgEvhN(yn8#)#Rtg<&9shH?5hf9o!CSXJ z-Lid8mAz#?Ro}XL-s*uF-`&xlD{rq(|L1y}w{hU@o%tE?CjC?;#W@XX`l;#3O6z%J zCmJnvD6OSmizrws{*5-d=kx1Ku>dj^PQpdGx%+=seGy$-HIQ ziQ#SGmp(Pf=4}Fad$51^kSN^iYR*G;7oG#>`o1?wXqU8K>5Gl6hkW zYF1Hd!`r#b{`)kWw~65Go;$1XrmR)-@5g-mES>@C@1=3yJ$RFTM%L^)D^FHEWXqf8)O{rTRqhL&*h{(UEU;T$c|#<^zA1dZ@yIue#6^w z`=2!4=4~c;d-c95ys<9XmbBz&JVdiD^GI=esEen-RT|9VckQ-sq&3GAG0h zZ@t?;-NWYXHt_cJ`yRX{)T;SC=G&WlzeySD;f|7e)5Mc`(_~P33~vLEe`T`G+br<5 zbX4!;_ms67ee&E(@f5bF9dUQm$-HGa0mIu3z4qE_^EMm2jeF38H}0YybjYmq_gg*e zls)+FI+-_SruMB;SZjFeJb8F$o3}aOt=rZH=8g05)s4$rx}B0bDMJdz_t(osK`Wd7B5`22a?|dE-o|^XHA#sihO7Cq$>?P3nqq z>OzNxw~=Rd+}Y-BK6q>Kcm>`;Edp=1Kbev@ zo?WA8en!$wn;AlOomC!954qlN`yV{p`TJRSg12Gq`)B)}<~d2tv?WhE-lS}dOB0ad zTpL5@%GxRCO(*ndG&}dzdfuc?G!+YL z&3ig##3!$@d0PVBwysIZo2CbiNBY$J#x=F*ae0#}a#d^{qX#{2A-gMdZg^Yx{u7IA z-j;&5r#lVE_B{>hInTAlQwaP-mwqZ?9%&c!P)q)-;(FfL6*W6yoVwsKy!~U74Og9pT39`f84cTFR4*o3tuMT+^*@UFJ-On{UN}-{7`$-=`Orar;=2x3kK; z4d^y-`+H6~ZE0@BQ{aO@Ev0ScIlP@$=55g{4Zxf4gf%DI zxV&k0M^8Sy6)GCuPFQpJEQh!A%e;;0dt$bndrH%Dhc|vkGt0EQqz-s zztJgWkm6ih(1y2GfBgggIq_{jpI%Ys?Y?_b@@Cbl_-7!oJ<2&$>`nja4yh7*0?gmM zU$fFq*|VNEX47a^DfkUE2VTR_T*ftGH(J>@Ar$-)ziWZ#q}ru70xV zd(QoKb(yzz4`jewJ)e%$TQYAU%&;PLNY0S8dfwPAwX(Xo-zFY*;VOr>Ys$R6w7Cjz z)*D^A$3o&M_EM^=**o*IPwHge=+sn{{ESme9kLevnm9ewE^kt1u99@ZTEp9whfeL~ z@OEvPw^gHuZ6EKcG#{QUJuYvOldBq0ySzo(h2gE)KepWM@OE99w*?QSC_wjqZQ1Z(inYOpD>}{ie)cI#XYN>=V@F zkH;IQL9OYEo{+W7ES<=k)oD)ZK1Tm{||a+VltbWb|oQk+7~ zC+6~&)!A9oFueWO|Gsl7cw6(~->+|h-{-LIkqW>`*S8dxHW1q{)#V_l>34zCbgS{O z3N_bK(?czJq#a4|bilDXH9Mp(cnolBjyU<5GH^xST7tJVt!{Q7ROZ=UwaZ(*?Ns8& zx6gFE`Bo|T4R7aM`QouQZ`XsjB@=h>*0lcFOL6+8hdfu_`mTEU&o*zZz}ti;D)7ep zQ_`FOJ(^D?{metH@z77LS)0t5CWCLq5I@&avrlSz((%S@)XZ#YbHDxm{|>v-=B+h& z8`f@w{ykOJEq$`skF-{bb1gOf)bvQ2pM~Nn_-3G=TB=A7wdD7WGbeY)cskyEJ1O`L zZ^xX~>HwR!HsGz-)Req&M^bMZkF-{b^Zy1x?OQAPev_()#Mv#iZ%(eImL`ynH)f+w z=+^MIdxs~Q*u1p`Z(Ub=@Rm@k;+fh58J=86S#Z#yU9xliKaV^G#bLH*mcaCgp^L7Jxd*ap#yeaR`Oju$qYH9Mm9*L-Y z?l-Ah-?%glO*~uPq|OVSnEUO`m)4$b^VS}`-Stcb-ZZ;tbWb|oBzo(48tHiBPN@@i zYIxh}pilL&dFue)CUhBv)KWhh57*L9?dzeRx_AoAOiiaoYt~Ahtm5pGIg@## zlbYF-+VFPC^UvI9^VSi(jhdB`H)Ri*yt$fM%Fke_pLF+|rjvT}B=eSGw}!XZS3f(} z=B*QW>+(WQylFl=l1GY%%*wsrn2i;w>7mwmBtO?u(@#y0M(4`gE4zHYsm)tw@YcG= z=ydNXC!o>3_m3uLh^Kf8_12ZNBlbxx`5BML8#^gzbHBZN+I7y)r(M9?^S9^38z&H= zxko9F##1~6DT5}?Dw5XZiS5zz#$B*0DNav88{VGnv1~v4e(MU}?tH1ic(e9c`hLte z9)7)gSTGuJ$RF*!8j*CT@N1iP&@^8NKGd- zJsQomS!vdla?+E`8=X=%W!&6vzdP;?{QFr&-g~$oWRvSH(U%Uh)LGQ9oYs>`podFuz>dT#LG zEmEHqzaI;!r6D1lx7ZyrzqF(O&)#{5TUlKD->CT|H;L)-rr+H3Vq#43rY9z*_n6!i zlbfiq*Vtl3x`f6#e#y03JSkHZl1kA&&*!$Uh~e(yZ1r< z`#$S@XRS4BK6~aIl!FxK+Cq67eC2;%Ve{4xyloqNZT5T0JCnPRH2u_)pK(r)YpEqa z;~HH)1y-bH23Du`^+-KLmNxg>w2epn&E~B?c-y$54sYBMHG5z@ zh33AP-?s<)-TRGIB+crQ=30%1U1|K1M~cVgO{!9g8{YOi_9*NpUS&Q57$!DBWe02&9yw)Y~Hx4nKgIBJ~<6NZ_Gx`%u>AMF}$6%bku*?_uD}5 zHv5|K?t99dl2-OmlQFJsly-UJP8-!#in@B#ylvQ7gE!Vnp_TpT=2?@!IeF7`?t3EPyeTUh-ag)L2Y~uHfpY-ria?{7~bBn-_Hixyo~{GgCDHJ8_%8jr8qs!prt=IrQEc4BzzbNKk{ZQia0Z%fzJ z;4R~gu6fpypFL|d^GKR`G#>gnxAG}41D({8M-$h0Sc_V-R`M{9o;N9*GH!T#`FFQG zYx6c9ye+(9O1}4$J8EW4#;BYBev36#J#T>(4R4PgdhlSIw+Y~F)XX}(No@P}#ypZ{ zPO?GGwex zcTJ*P-lWcbJGA_Ux8WyG`JBz$H1M|T@jAS*H|~OQYK@2fM(N^tYwS)*ou-o<=V*BQ z>Ba@aZQiDXw}<;oZ~pf*$KLDSw|jZZH-U_^FuZj;X1w#?PiKI)n-(@HZ_;NlW50D7 z<63vx{Hj#FCB}KPm34}pmU0^2uKML^J?;B#CU_gWIWlj)d-3(hI&qh`SectGkIP%4 z4h?VLKjp8RZQf>qw{HDsH0OJoXeO0Aa4&DsCt!HH{U_s{|6V^EyuEN^Lf(qJyQErD zJo>aNB=eIZRi`JlF7$qT_vp_)Zr^Wnz}o{`67rUMk|FuCc_Sy(Ar*bs#;RK=Z!do8 zys{2GmlecY#C_F#xqMG~ z&t{C%$mWesNh{CJ@V4u)?>%PoHV?dA@q9tNNqZ?W&bjU7&7A<}ljB?rZ#{Zu7PP zyxo7Z2XC<^VD%Y=AK%_w-lPn^j4=;&ie0le)}qeljZRI68Gggt$0mQ<`TcJZ!58ixt%3U{`E}ue6*-=QJE^i^( zGS1Q5Z*M$%%XXW$8^PP!QFGk)G*Le_@3)Npz5Ki}H2s%PL8{Jd8qGYCpB`%ZHF0{l zQz?(j8#73n*(7bPHM~9dUlY%XKV&Fs&|TaFzW z-hO*bzdkl^OTpXBT`ib5-}Q0sZ)svO$zwxG?oAFcAXcK?0zTV;O)GBYcPpiRO zNI#8QYwf+}okmYMZy`HnC#iKfZ_1q--tIkY&Un z@2**w+Bc8On{NjCGdeN6eQNVJ?{#=Py~vSCzL755ImsY)qsc^!y3$6JoB3~w{qpSa24?VKua&s@2%`1dsAXIQSK_MMNOH&&t6RHQ#dOSPn&^y_(( zvPp4PDR~TUryPAe{`vad@9DW!-X5Rd0N&WAMhE^g*ER20J#U&0>1otG#vZ8kym3_y z{p$U;edYU{`|aOV-fn%O1@M+*?==}?o;`y%4cpngX)%Szc)PgD z+cjGw@K)sXLKx%Pxc3_~OS(~0DwMY=6aO*A;jLAbx67`+QU9JwlhnjxO*>Ye;(60d zDTOB=Z@zOb`OWulUv2Y}`}buosq*&ZvV^=z-*srFs#)8r?>Ut}d!c52=Aq8V8#7R| zqNG#%rMmfe({yWj3~$HZ*t-kvw{@@n{rcAUeGU&jpAa}{1S#Uu2U3dX11?49sdMH7 zE=N}exVQ8%SISqQK>7k~F znjVeL$Czdp^n`RM?Lx{a#lv}HW@@WL^PaYT>2vrCTIKh&J$P$BVrjbfl#{o5t1y{5 zF7;481%8z}byrgUkRGJ9;k-%RGG3}~csuvPuaCFyw+`TK*wHtodrw2AlwvJu$;Vsju2SSLl(!A-wq9ZL)(O0IziTgV4fy#q1y@qeVt6Zk zmd$(m!28BtYx8y)czb?V1m1XWNV^V+OZla^rdIRZZ<=mHJjL)<>d^4^!Fl&QY4g?@ zyxli$*=y%L<*7B6H{VZ+uixD%spD58YQtOGvPLY#!UEcUrjb?|`)>^~c;Xi)!uk7#Nx`MZM57gn!%31lOYvPGM zN0lBz@|RD+H-oQVGbPCrtCq`~lv#>P9a?e2+o_iyxXb3P8+hB$afSPy2G*_cVD-~f z@qG7NGw(?1P0yQ@K^Zr^wf*i#KVb9L9lYH%xdv~_{JHrd&(x{wGQXa;Ms>(;L$u3V zNVbf%hPP|}=Zv4(y!8NY*RH9<8{cbc)|KKZJUo3(UG_k&=Z&i*t?5MLk@8D%=A@?I z;zWH6=d^0Hh zhPPc`IQvsJZ&!l1SEd!j8+#6!9`l6ge7s2&n_)*%Z`r&>?A-8n`gt#Iwt4FX-qx>g z0B^j{silcAPCqq0)RL!s3jC@xd3rRO)v39bagF9$YR%eg-lPmtm5jLI?bADF&9Hgv z4c_kUa*O^wl}<$JIaWMYUFL~Z*Lm8{8^i~KHzQkEDzqK z9ZB&VJ$rpmDAm=(^}IFnF7!Jkb(&Qg4<}R7hPMOn-u642x2wS0pp729W!$B8IpJDK zOM9fB9>z7AYs>SOR?$fh>-;sIs>)gZN(qo z+Slf70C-!|>o)g2)$CgGNb$hE)O?FWayBP#Av==R#^o*Jd<}00{J!sLHg5yL+rkAk zc#}Gp;u$@7al$;*zI8QgTNZDc9Z6G)%bQfC6gRw0yZ<-%2S3XH|Jxw&HvGv5yk+#@ z#aoeiT;4*?G7k}4A8%4N#w~5`xAz^n=t`Tn!QgGzqB^{>^F2wsyzNQ5<6iRd z#tcerc)RG_&+V{z8v@=Qefl-cn{;z3C+u~5H!5c~ZxJV8cpLcO4NGm_hJv?S2Hc+g zo;GSqsp~fX{T7RaY~CWy+3Z}-|nLi{C*Sfs13N=@0G}o52;q6~rzjwaP+cn^A+_O1()ASR|8GZrQCgZtIB(KU882ybza4z(kij-@qrlsOVRz-H9e)ZhPNM_@yP3K z-bRDBDYw?)P3l34vj^%}e&*rcr8s+_c6sAhshOFY9%^3?{nX4!t?_U&dfr%t88q=y zSBAH*+1G`e`+G+jxa@ST#n6je$&4R2c)&2s*G{W$Qp`Su#TNxReBCtH`=_72XD*A+^v646ZKj5-PBCg*Q4jH z$X)yHLgUZoP1>ClXGP0ncw6$SgWOLmp5yA4fzzLn5xTLiYld? zhPS2PetDIBzg-93dbGL6z27vGPtU$fTbjjO1+JH|Ddol9Di zNAfVI zx9?Aw=={E#so?Fp)irqY?ZK+sBAC?k<~xm8e&4A|{z7@1c*YN$-%maby!GsKul_xi zcEkzOqq*(rp=QnyE#)kq0>4Vlirf+7zFNw!$rG|R9dA}w=6;(#_U>QX@9A{#)_ST3 zZ<@)p6i>eUEf#yR@@S@=jyJ2i;qA|FefVQGZ!^H#BWvsMW-(m%t(5$XOPc-D!#K6? zsWGQ}zcDklRE62-_tjD!Dejv;n>T59Qk9IZ3~wi{czUhP+f49w&*k^&-&0Kw?1x%f z+o-snH>pGJj+%ZcP7k%_^qG@>#+lRQO>%No4sCe5_>N0xK z&&Bhm>6{+TF6fuEp0~h?hPSpGK0D2RPv?NQ%e&nl{+>!ZO%V^ANrlPsDa6W_VyCfk z>Uj&SXn4EnYkzy(=Iwg$_RQ=S##`V_EZ&$c1$&yDl1GZOXN}hLCRLQ;tX}dM-md!2 zl6Tm=%>{2OAKA;>?tkCenqGxW#+yNFPM>}1dE+YID!X}m=L286&gN|%c$?DmftK~2 zN;C2OnX7yX%;u|Ek(zmUYLbV3YRS(y^K&hAHgC+vRZ^TDOPl-c&X1hC%;s%AcpE$~ zCvWVB+UlWcZ;5c;e0Qp0!#AhKpUs;lgVJMoJLd-mjI<2iX#_E49%#-4&^ z7meyr)0@UqC~sSOfATW>e!Bs@ZSK9gdEV0^Pqz5`O~Z!9<2$8%y!losb#CsrOW$`cpZ1 zvuf33i5}^DJjSWHHia&qLXj0y>_Y0F@j`if<5y2R*5++7cpKjD!Q$W36!(?yX{Nj1 zG`pjR+IJTke_Y-&x;6LP=r{dhi_P1O;H~401@V^AXWhF|GgV(tI^Nhx>RoH{>v_{; z|-^@})-?&s)iZeew*}Tyy zWl(wyZ#RGOqT_7dmV&p32Cm6|PXn2+%OUeqbMI1|9$&5JO_N#bP>P4-)U2f^n>RW` zx-I1~ygl~!IWOA0-2~olxTywjfpzOL#y$FKsW&MuU8QPuRYXW*vaDWTJ>(s(Qe z3~sOc(GmTsxb0Ks?f5Eh8&^c;&FUdKOR@47!&{D1i#{=umc8r1ZVhj%e{#%Q9Ntc- z^0wlI2)rqOe5p)Q@=GkyuhCK-#>$;;3Z<>BuqDOzfaoW_)yleL9 z@)oOeRw?B)yxsJ}(Z@Tyol@m(%gFVwo%fV;*7L?y8qHIn)~rq8NynSjt>Nw9AI))p zK0UR{+p0Sf^2YCGn_=>tPdeUMMbexMwdD7WbK2>6W0g{ehPMZvy5l3x{dQWFxA{A3 z@Mh(y`~5uqoF3!UoF4s*OMZH&nTMJl>hdW_PEB>LrDl~BTGJcr>UrZXsF^LJy5a56 zpPoL-;qCM)Z==ROwD&y?zw0tiq4H^!Jic+BzNDE`&zqEuab{+`qz!Ms?mhDbhqp7T zymh*#4sXmaX{&B!!kRd9N`8$;S}VoF8Iv+MTioR>WS$vo4R2o^{SEv&{N4Y3;a^qW zHnrK1?tYVclkVyiC&gOqr+fCtHBq2^ldh;eF{H-1&ptVm726*s(% z9r3F%&i!^~mACuG=j2VZQ})(uI^X@)5_g@Cw;bnXcpE=v>*Wq_XH|Kde_u}Ca`aR4 zoh|YN`Tqvw<1N;%Q|IAS3+3&S{hqkX;qB}yZ^Jq~oc*4%bIp{jJGLfcnmkfnDbAc4 zPx%zsEj6=gbjVsw{t!<%Z_3@7`|ZzPyZJDOw{xnzb)Hm*H{YG|4&ho21N5iR?7_X? z_*JR8#v`p|JXRj&lyo?6Qf6xxhPTO2>~Q~k*11*Q9$)Rjo0UKJ-6i$Uc+v2 z>2$nVQ!>20v)>y(;oNWkuJU$Ur;XX~sg*M+V^S^YK9`u3{LJt2rdh=dQgw~Tw-$49 zEj2yV*}Q2oXm(nvXn6bEg&!Q~@OEC6w^>sg%$ssAUYu)ky1em>8l^RzYkFgzY~D24 zlpe#|lvclb+2QT{DsKbVHJCT$UJ~)fb7@p3nz|u<7Rp-C- zH7DZhX`cHngeA?cvw356YG#(=^pv!@-|kwqW1GXIT;=WBZjToKo~E9>Ccn#@W|i+*H-kT%H|1o^{dP^S9Va=wwW{*gZB7Gt zYo1ex)rsariu7<_;k+?hhBmx?ecQ0_IJ{j_&d16GLkXjruvRT>a$SZ zhW~lUC&AmgSO0!}Yy3WkRXrceeo!?ZOKGc8JS1nxTK3~UsQhYb9cvewJ_`lz?wP;; zP!+f`Z*9Qa?0I!~)9gs{X#Vh8TAMna%^N$X4(ZA_5B(a=-spFEBkgob88q>d$2_Rx z|Mup$+q|^}Z{r_N$Q%2l=6)HMwB(oKQXa;$d80F=3iJ5tSan_Aq)q}W8s7f6dfsH4 zw|3yITc1tY@2S+AZ`^k;zJA|+^t}0IZZ^N3x4_N~Z$Dl3vk5kD?ZMk~HzeduVn&Jw z?$_(vcbB(D{lMTm4PU>@n{Ni?xftFi|9ki$Hg6rk+ucvq;4M)Y&YQAZ!`odO zes-tL+hySGiGfe(-&5%>mExKcl-4TunEMVapMs`CsS~O5kepJjka#w4?3P+nk^a&& z3~%l4JL-O$x6a`0_M1F-)AZ0%JnsFb>6{Zz;n(D(N7GL>Z_MVat?Gui{jUD*Xq&go z!Q0g53gS(=4Se?)%U?bPc17)5g=?vO^ThJAhkU&G{^?MuqT%h$lP=z6^L7Pz8#?4k z{d>x3SZr6`H7U-u&7jMt&{DeP#C&_s#~Ujuwc)K(>-Oty-nxLd_BZF`P1!?*!)EiB zPoc=Jd^w5LTcNy7{pqjP+PrlIZ;!vw!gz~yYQ^)W;V0JaSSy>ih!xHK_DYxE-DdOF z4ZPiU%~Sc_Q|`!D`|i=#uj!{bdDC?6>&fQLH(Q3^@OIKC?)_^A@plbH5$`yWX2^-g<+#X=9%*{yhz0iJwPm-?_QGg=F)sF8O2iz&vqz zV`gg?hPU(XxPPq8TOaT??4BCDkq2rkf70*gQ`d5NOI@*1`Q!4Iu?xf7;XRJO$L8%S z@YbpAGjZ>?4F0|Nl=4XNj5$>5a&0zmu|8$2NX>2;r>2Km%^O#>bgkztaHod1!QYsAw|&3$18?i^ z^WcsB@a!1()xLRr{Zc>0^QP%sqm)wc07X{ra7fI?bva9>d#rK62>SZQcfew~>>M2;# z^iR^tTC*yL$MCl9^R4j@XqA7T!$9!X<-t0<6*(nMEqbVZ=aY2uUZ04NU1)Z#=Pj@k z!`oR$|JC{X&V#_)=1yB1`<`khmcr9KyrtM3bLx5HDrzN%hPV4)Ub@wOPp<}VcTH_l z-lRQBan47h_iWxcrEuOf^DKD`Z#z0%?EL>ogTdR(bp`R3@#i*`*N0Q$9vPSNFwV77 zJRNVWNG)|GdFT((*}Q40(Z{x0b)_qTSI^)#zP}5V=hPMlzc>MD=Z$rV`sF^i*W6oyNyz4@G zE1yEFY?@Arti^s@-uP9GR(5W9TQ~HOZ`-^L18m@Zg~uEYdXK!-RA8Y@V2enbMAX8?ZV1mmn-_InV)fL zdQ#}}DX^liPSKTbPN`?+$>xpDKy7%t^|Ozxuz4E}-Zstg;4QG93ZEgL)>u7c-)~YC zP3Ibqls`q>+Ag>c@KJ2Jez z^}WA*kIma?@Ydmp8oVj{Hw57dd^Om~0)K!Xjp}ajav;P-t-o}8pC;Dv5eosRt z5bM-p<>6kkd83m$MJLRkLKn*0C0~4l^WW>og16fi*5OV1{*aRiIR$C06lYyYvuA4N z%*UHlQ4?o%YOa;y^!VyRdHZ_Dpa03e-^PKrDVrnk#tHCz8J9Hu)RJElXC7*QV;D!?>h2=`k(yL$0 zeovWyPtw`H@E#=qb(X6FM&l^{<6UH;N;q9E~_uXOhHVM3~c|Ip^?58E_Y~G|D#o|qq zKcp7dy1enLQYSg$hPU5c`Df?%;ZFu{3x>X=e^1#bb&lS=?!y%M^YJEi(#+F{%Ui}y z4R3Eh^Zq~E_uCZkcFipwyh+p6#HD+fYb6ik^h;XfFP{RP)KX?moOvWI<&peSoOvXV z%NtkuYF4q<8r~KkbKyptx2fQ5&`Tb?Wz?$NR}%kH-0HcKM-$id!#wn7-*1|3>7i!D z5Y28idFW?9E^l;7ndxCR`mMO(?eKZuc-rP|8hC3rV!Qr5Wi4v#8L~EngzWpx zx2up&d~5mo)A45Q`qlUJg@vE7d7A;=R=pgVH>-!Fca-ns=`V)2W|&4g-Z;;aHoU#R z!zSnN%gh9C*N@(j{hsp1rnYK%am75;QcjJ>_hen(G*#H4Z$++^v~Lem9>%kI(^S-W zN)-)nPo8t#AMN{X7I+(XR~_Eid82fSlXZD()b3bUvs2~iS8jl8-ZT~2VTLxm&3)Iy z6K&pRgSQ^7U(UYYBKA|6Kt|6|Jer>MyfGVfqjurjLpE>HE~F|_yp+fA_Mx9Y*3;%~ z4tRTJTn*kbaz^1n+GCM%m$yB2*ZFwk-_%x~i{Wj?f_r~q^L9OWyK7Yq-jw-UG<)S! zNX=O4{F;+;d1F`9nrs;!!&|4%&wQWF+g$KAqundn?`cN<3U3j6uH@9z4e=CnziB2z z4|T}yxYp&3UyY?pog3c1`_sb@v3Z*Z-uhlwgExLh$_X$o^{MgG!#s?$7UT5$YUa#$ zzp)c$V<%Fao)9hNXPi1NZ_I4%!tl25;%A)S$2A|kwR@ljZ&selox^EM9x2W~sbl%2 zJW|}f-y~wg9|s>$o%fJ+*RH*z@g; ze&!6(%t_6hj8kXx#ta(G%+y>PE6$u6t>;a1pRgi#RMLjGBW{0jjQyV80N$RTT!%L+ zrx$O&Yc>02PG3LQX7fg8EG=~_#becVd5e|V%42vtbNK;pw0TyHPqB2jYij9v3#r11X}Y4HI%I7&Z_@5q zogQj>O4{6SKR@We^KIT1gSU~>Bk;x^sI7jI-dDbLUEX-6nr!rQr(8=dc^Idb^3$Wy zE^ks-%$CDrcw0PZ#VVV(8^K%ehjQ{3YZ}~Rk-B^eEqxc7iMhOKR^{**-j?@Xd!NnQ z67aU8OY8mduMoPQ`W$tW-ea-)Y3xpm#9}eVJmpi6 zb|J-?A(l>Et5Dv~+_wCyHg7k9w6Ro+%EXfSX5gi`ySg74b$DMXxh%}-G3{G5_B0hc%5u9%tH%42wY z+p*(5>F{<^mABbXM&^zEa2nJa5B(Z#?Y-vdGf%$zjTxw$VMpwT+U1R3W#`oND7E43 zm4E*3I)}HDtGtcq+ivfB8karg>D2v1#LAP+TdXRjJchSt9{s}zhqqIzymeXJM7;6y zMJ>%lt~vXH)&$r z7d@@;AL6L#k#xR0jh#^YzUEvjbr{Q|=S@>l>b4X&ysaDl^{XA;POI{^bwGRld&+*O zr5-e=pvlAh^h^2ak=EwpjoCu9rbEf&Ti4}Hnwyka(@Du=cpKCEGq*duonGZ_{ZbF! zq<$Fp-3!;o(wZI`%Uj4!eg9&p%UekGjJ1ZhGk<)@8yw!wsPeY>*~q+Q^i%l`S@U_3 z@^CG=Vq99wc=;5#ikePpdZ_7PSBy(q%0s`DQ(DV-Hg9yYiWFy8mNvZo=J*@kKY#mI zmAA=*JGk#D>smQ0%t-N&wNlO$arb`n?cCR&Vi%g6*}Q2oXm(oi7~cN!txtW*x!=yL z@^>8s6^y#gW|{-p;D>wryKZ z-U558drxv=DYVNQUl2{jW|)AcXFYGsMy;7f$zyo?^Q`0k;_!BMmA7@nI=b(vGJnIF z(ex=%;qoSBV+Lyar8qr}(t6%PIxJ1V@V50w+qXEpom1s);jInkE!Ii-=F#&Oatf)} zYIa==Z>60Y-X1-2R#%6&bE~|K+TLK^G$$fWRf=nJwp`wPcdh5mH?!q8y#4yR(_1;b z{kzIr=aHAT-1pSluh;wDa(UByn&{E<##K4A;q9WItiV4aOJg>^zmfLIa7IKa` z`t;%?R{qyAZ;8&=@b-nbzHPR1znx#@?VeX^@FtyUqCTVW=G$9y^5(k>>7*Dh^nUwg zt1JHQ@OD9!x7lMlx$h}|7N+Jde6=)5#x=Dh5A$exE1!atO_RabBXuannP2Kt6L)#z zDr#1gbcWyDZxeph`Dll?3#+`1zPko*Qmt5VX~I%mQ&+Q=9`}CZS0%0Kq}e; z4>368}D0c&GbS%tVJ#5%;t?5sHJS0IP*|z*3!eZ)bvohym6JJnL()y zZ`d6PilH3ZTSsw(>neW|3i71w>IFd z)9MEBCiP!rJT7n60OZUv;^a@-XgBo?rFV zDLRyT@U4~28>{$c=Gu}rylvm-fAI$*W!~C@x9MwJ0B_d5>hh+UTg&CmcN%)$d^1~q z!`pj*cibcP{ni1z4Y>S@=6O%8{_Aq(>nDdUZ~Q8CNEK-<^HWQH-#Bx+yz#5PnKS%` zx5Y=^{x_Sqj^J&_^qjm&e^AJHMsF3CeEpi7^ho(#-Xtfp#nP-LX{na4-{nopEX6ey zOCH19g}b^wVDok2``Wb@Vu zyxrHeOJmp(}Ws-n*OrJqjvKX-QdBSW_K;cQw(p-FpWZaYde1Xhi%@v zgSR$My@q(>?UBtJon%Ie)6;CaP~Q5zeabU7Z#}@6DM(#uxQgYGdSE=H zRyy9SuFUsu_x6k&b1m{J_T0MXia9x!~FC}ae90;a~8u} ziG0J`)4dO=;&j zyzPAKvG-PiEA!SHyzN|Chqu&I(&Xo-m6POJdZ_8AcJDWSwMflwxg%<+AFh==E^nGu zQZ~k!r_`=dgkYXTKYuOBh~fw?0LLN6Ci2suBBB{T$+C=Zg{)vy}x$;Jftspo429{Z>&X~ zQLFCHlFhcZJbCVp+P7P-4bi@JB|m#`dE-~5&$yI1BW`%R>5TI~W8ZK6z}v(ZYVekk zv&C^G&C~s)hwNIa5|T64T9-E|bKouvZwJmi;Nv!L{lQzWYpzW9o(A@xn{7?qxV%Xx z*bH$!Z-Lz!-p+mR6#N4-<)2RnfVb_pCgd%!|J=N3>b{nF(|jH^GqgO0x3+J6sk?o@ z4FqrNw%6c|@4KX}S`{|vmpoEjTFW>!^GI=esIz&KPDHB0xTINK($ZSSxt4KBXWwsJ z<(rxQk~X~E{Dvowuz4E<-d2z7mF_*Y`l&G44DoEuF$Cx*8Vj9YxU z&D+)BZQ&h}dDEPN#uIT4H8~IGjU6__j@S>ip0~hV7~X#J#S!?0s^#zB27|YeJL~Wk z@xHB`J9Bc6jB{eXnt7yu_&F|bz8(7dxg%;#9zAcGZ1h;27~cM4-jY7{{Wb)=^&8uJ z?|WK45t7fT`s!xOla4oQGKRNvU-;x^o429hZRgz$;LZA6RPH^i|0r?RqIT~$ew8{! zSH3yf4|!%>%IWf^StVu5SZjED^0c8x*}M${Z%?)9(>(W^v==EZ%|sJ-dDE<7C)8Xk z#p$7@pPC*?(=Tb4Hz}JGC*hK(6gRwG`lCOrwRyV+ysa5uhc~H~6lXsgPl%sueRV$G zq$<8UmGY#Bhx5j+GInZsd&|WKe!=E#ICz_XUk%-Hn(zb}jcZ^JuW)jaQM#$KXx-u%49nnt0#efXGjpRn(@k>Ks}NeOw=+%vJJ z>YHaTZ~6%+&(Yj(|9DrYhiu+PfwwKIYw)JbU-!-l$s>LDORnf)oSGi$>}k`oSu?4yuIhv^Z&!Gl9_mumhw)(8gt>o9_lsx5A zkiIHaWL#P+#Wic`iKX?tX)0#y&hR#O-O8`pyo~{GcTSDK8$YcfJ##PA{BH!*QZ25P zJlXe~w5p}VIe~26==42DYZ`{P@Bi27=i0oD1#gSj<>XD%TP%;YuiW<`^XPfg?1Gt_ zO}o6YLuq$XoNK8|e#6`NA0O)cdmrP#+wd#;rF&1=PqS$~Z_U=ZrXSLh&6_5h(qnl0 za?f@*+xOeG;O+96EsQr#QaOj3_u-z+8z)mJZ&$6F*vsZ^Jb2r(p@s3rNlNVTdlLGo zi{Xu(P_vT|%?U{P3+3(aYyPp==4}Fad!SqYxbG<^5HiWsYm4D6^)58iE|j+e-*xK# zHgDH~w;Sd}<}LMvW98p7c+*THoHq^CrB2NEZ^sSka*fT~MDRB1(FnXL`>FYTlxAWn zJdNe8$ccsXrkuawt=qN_eaz-<5_r4f$^q_sTI4i}ti`!CmN$0dt9>Vy&6{sFsoPQ> z!`pewZhec*+hp*zd0q|Pn3Mfb(-TXl&d)yeys;~4W@~1h%^N!p+@0ZV((KkR+PqBx zZ?`^Phd1_vX(XlQF!l*){W{HgD6x+s>zI@WwlnS~FGl;JcQ7YSv|(wImPy^km;} z(kf<%<b*D2h5mi^zUXh9ESAc@)nXUqg%t<gU!DJ+H3z(PSXP5K>-)gUr`30-sr^!)Qk?y`yz#3Vt*IjA z;aXoNxmL>0cskzLd8sRNzjazU>=^rgyB@qf|6C2;*zKOG%ctObcCn^WWZiVUS<^7Q zJ^#CGh4QxGj*ia#HV?ercykTj8a0{J zb@@5U=8ewOUHRts?X6JWMzuPAlzqR=2XA9utic;^durdEa+1u$I5j=g^k_8KQnP=T zH#&W*X#Bn&-+rY0dft31T7L7Mess?CTW#JJfVTm|hq&*l)cKwicX^X`M@l76GsLAn z^}Ge{!tmC0|8-~EyxjoaTCZ#XZ=4LZv|oDonWC1~N^w1JQU+G1mir{j(1Qrey2 zZO0LRcmDh7Lh$z3jvBm$Oix58-|%+*qt{<$^R@)MbsaY>{5{o7KAv&ZV*3IUPPG23XLrq=QqUKsX zZ(Nl_8{R(o=EI!dhkp}zTiNy+{r$!rN!qth=8@KFJknYzo{l%EVx!_NZz<+k+MVI; z(ABHoWZ!Sgz}ut=9=tVbVkzprmU+|6GY6A~w@bhBov&AUTf6_h|9tzg#~m}V=ZbDK zjx4=(eed6kTaEv{S92h~9^V6`pW3&sd%gK)Q2GsSJ>UQG`>MF@Q|9gXDsQiJ816o& z+?}MAJ=Fd7jrlb>H9s~nkCeZB3es1ZL8D{k(bSSWoD6lLynXC9N6m70JE6+kx=A&7 zlXlcn;+oUn`7J*yp6l|f{Ps9PO9=Y@3Il@d&->D%uh{^ z=8r(6wai0JzeaN{HP@C;ff32cI5pQ&OCH8GT9cC=NxQsBPEA+zSa}R@ue{^qw>i9> zT;*-{w8*@%2lhrSd8D|sR^#Vd>g@YX%FH-5GdG*g=1uC>cXuVf;q8|B6K`~QJEh9o z#PuG$Nxf;}(rvFAt5Lw&|gk z`lm;lBt6tpPTx3ly7wEiNgdL|Y}ArRigPVB{l1!cT;BLqDYM4IwI#pdZTRr^a~Lvzd-~xType6{M19u% zGdiiRZ*TOMPk|AsN=V%I)4_hILway|V`fRSB0Y>-+VD2@=|i7(csrxY+xqUK-1n4u zG!tV#)XYQ8oEksZ($74SM~Y|PZ>&PiY#JT1mieWf(j#e?H^~_oH@wZAH2!FZw|`Z6 zyY>1yyoK}-GNq8Znzh-yX)-jMN1Aq_ylpt*%EKJq&aCn_<*|gkHQR}>pFNK^?v6Sx zZxYp|ispVh;*IZH?eKP1mAC%AMrXgL>^wwg^jZ1-EtZFUN;!+?P3nZ*rqFSD%jnSX z_TlNvzvu9Fc9pkR=GWkj-*r&47UR_PXf)Sqw3L%;>5<|tZ*)>i*(4AB)Dmw}oNK9B zmzo}GJ#WlL&1}?`$ME+0OUB`!U)cTshn!R8ZQBzycw?PvfD{r(^Hy>}z=G(2WpS2{d z$(hZYlsQM-@YeReCqCow_U|fha~3rbZ#ni@mn$i!%bVuxq-?R`e4(gA@@MlFt4gUu z!`qDOjvnFgc3zdY5l`>sE$9E=OFpSX*oc*fJ!JDnXNWF!Zg{)l?Srm$cssw!+hqgB z>fh6dpM}c3#GKU9-sz#1{8GGp3YrWl7?8S(73aQ+;jPrU;q51nta`J<+XYqLURY9x zH+D`fO-AF9))pBrpF&8*mcn5+Z@%Z1;WxbX=<)YAIJ{k0#zm*>-D?;JmjJ(Z#P~&PXC^2dT1%0Y~Fl#VNJgB z-2(GyJoLxX>3Fj`G`x-Y=51#>_uIu)-o`C&0B?<&Jm-_mTQhXXZZ%r!xlrB)Px}#m zJ<{&`tyPt`Zrd8bn`U~cJ=wgat{yAD#2Dj+@;3FO+qXOS+a*=ro*#Pc-uKjHkLMMO zjYj3k$6JmGnD_L&_wmPteb&AD_v>5Z_c`2uiwAJNP0&vrGWu9~^pLS4HFqJ!>1mX9 zflJh_0q%)Uzx|}@1y|;+4S1XNk_T_n#1i#i^V@Ez2Th!Py1Z#Jgm_pzB#&k-Yq`8> zGH5)kQt}(#2JE+Pip^VF@HSw?_;e4d)lYPOa?GUCL-`b%p+n6sLOj{LX(pigOU1i+ z`^-M~UuN^x4!pg*(t|htVIMW8!MLO)e@L8bsihv=`;AU!kTm_&^fOLPkFWO47K?lAOzaXjn=OLZI+kk5$@TTcg|J@m` zrIzZJPl2CMuA-JQ(8F%&XMV;*H0x4JIbGf)C*xA)j2#)?zPe9`w)XvY8F=e)Zyny) z12y}hrl*;8_Wh>WQO4w>yr=Aenp2hH^yqmD+>zmJ!RKCgxy@T=@b*f(iP`TdJ5Qkl z_n7-+vw73(G^F#8wVK}O(euVtIke%e^^qNpw0XN6ysf`3GH*prk~3jlYI>-{d1JO% zTGP44!+!Gd=3Avy(eO6^?6+F~9_8DQy#l-~e4qwzyrm`09Z5Q5Epsvt013ogq)pLPXrgQhfqx4?Z>m%I}*;@@MlF zvAW@{%dH=G{(Dw8@V0ABLA>#<)@W&ZvEnXou`)MX9+x-G{OM8d$nf^*+ut_EeowoD zx6Pd=yYHz~i*ai9BgN@y2JQ03&Z*fg*^4oYSLb9>%GCJ@ixSdE=@a+VJ+d6W)Vg zI9UGvttWW9`=J`VNwqi)YH9NHP%}T{)bvnG9>%G;mVN4Zld>^R&59b$wbblZlSlFt z%G-|*`sxMt{dOgITh?`o`<_ajH$yz`DQI>StD;nk`Kcv8;~IaVyscZca(|n*Uf^xq z?3}#ulSs{JglNqqL-M=#TS&Il-G=m5C~tE=c))jT-g<+#o{!|@Eo1_|Yg3$NT;6i5d))`SRB_w=_fPwRw<&e;(2gyp8QWE#G_UyK9~Zf4-omN75m6nKPR=->#Th zTC0gm9_IJe`FQi~q~tg6>F*9a;53`J{@|_O4Rv^9U25N5hxlVL#vZbHqq9hzZl|O! z(T)snx6Jq^enENpej5PZ+B{Wo#iGK^D8c{Fhi=Nb=d<>M{Zj@Svcl%Mf@JN4zw zw@PUjhPMGrKibQ_-v)xWNBU0Jzo*DWD=R5yD(qQn`f8Y%LdurwP`V*%d{a5mbtd)A!^X6OK*DrM#E04>YR537ac>C4Z zr#@icZ$rS_D@#3iv+mCd4~$d$_RqDF*5uTzEuR9=bo*>n{PinHL0Iiao^tPr*`i*epS>Uc;Gd~xFP#;&MK(=hkj4g0?Hi#Bh=!Q0#yYVgKxQ%^Nk ze&3#5-ePs4$>ZCtCa2_ac?+qYvDWbRgNt7`+2(BocpH4pEcZRl$Qk`zMBJZm$wvEN;wU0kA3L-f3bNR3EnQdtp;z2`l+z0IoZ@bxV*7qGjzy) zLUf_LUGmaj|6%hs3cPLJ?!g;B(bOT+h_#meNSd|Er=W4t!{56l#pz*1Ni&b+35gfV zTdRxCckZ{*;O&u-v)%VJWOprPZTS@Tq4<~v@~yh&%&%yHk( zNTIx)``nW++wbXE@HS5baF=$ z)snvyH$N}DAj9UJKrO-?Nvu-G5(3GkJ|C$4OOwa!;`~o41G+4R4*l^@%0+dpaJxZEtgZ z_Wj0BA2s`w;u;UvN;)JD*HV{Hfe~tnD|Se~6sJd{nTJ}lR`Te1V?}Camf|Ik;qCJE z8$W3CHUYeC8SlXx>-y@9K9hdBr9QKHlU6a#9Z8ygYRS(y{gT%BUEVaSq>79)Pe~iz zR`&nXFq^mQz}v?A>hQ+=(x;vtNwYV}lRC~GT;8}lI;m54m?FQ+TSyg7gSwQ{@OH(} zBe&YTO$2W%JIu{~PuUw!gmFpx`k5zWttL-4Z_Gd)t1IS=rKR5Vyh$099U9&?d}ZaG zHgA)_+jWyXc+=Dk=~=T@`=-h#hE9T*7Fuv(eU=@m#-UY^EMg0 z^?5KdZ-G5mzL(>v#mdP%E^quQwQncB{*YQ+E9rc^<=BPc?bQGI#t56YDd6p;PV@Bd zsk9@_rkZTj|t7a|M)#!A*h3r~dE5-FYC3TurIXs59$KSL4aGSU3;BDL$^TXd$ zsW;xw)RJGCD%bjYxR%=GjhX3`x|KXqT*?y?XDw>xBxz+$r=*!d($-qT+rr=c{uP_I z8Q|^enGty7NlCqF;<55*YGv~l(us8nxo>0N9wfiZn`S;aJchUXcOL5ezUZ0YtML% z^POIyy!GpJ_z^a5v%%YnIW3qse$pk))1fCHZ_FTNW}I4DOHYVqe(FMboBWX*xqsS{@<^vizs5tqM(-K-n|`OHPP0nmvChTtcGRZNTw(Jz z54>%9ydd8AdEs8Ho@??ftz{lxoz0u3D@{e_q1LR83mc<(|)*u32c-sTNj?7pYupE_c1UVO3#=@g{6)N}b1m|1c% zGqvQ`#97xjC)Y}~=+X1WRZ`}RxZ!QoG1E`6d0PVBMlY+uTSiVVPMSGS@%LM!rlIF8 zutUS!hi?D*-8OGa!CUv|J$Uo|Buf4J=AmEGzO~Zv#tx;fQpBaYE^m@Euq(sclQ#~( z+2-vg@b>hO8{PLbux^XE1Jm)=Y}0djV}}vhGrV2?@{~_O`l@d> zvgeyqs>L{S`qpxJV+K~Crbntye@PqOI==Iee^sxy-MpPp<*n<=I=opqE1XDi_9=Oq zAs&9e6}gL$sk*!+>fG@5tIz*aIN zhqsfeyge{#Y4&@{&V9A!Q|arWpW3&#@+tUc(D-9{e0L=I3+1iff8Y5Fhqsffyv@3^ z4sV+2Njq(ZxYYmN`z`te%zOIG>96bO@ODa-x52yW@YW2>r05~~w4=QE(x*e3fW}|w z{WkRD?YcO;om%Cs{kWU5-&4(wq-li2Q`gn=mb#PF`CZ;L6*V5~e9irK(2jQQ&kLtj zd3)rZI=opqbAO8YGc|Kcamho!q`8)w{_-g>LZ`3JaXOVrvIlBS53I|z)GlxQDz&C! zhR5*Me&YYT-MQaRukv z{q#2--p;7hl~rQU@mzn(WuHhTC;DQUx7*W3TG+2QTXDsOE&E>HKK zvLE&tONZolc?-!FvZILKLDl@EXY&?u*M_%u9dHAF9oO#vo^@80w?`(`;LW!OUw_Ey z#H!`;7SdIbYo!S=PMytLNaw6l%42vtuE!03aPGIWtGq3Fum*1td#L$|3&|gAZS(VH zoo4Py=Ho3#b;H~H_WR9Q4sYjFc^h}xigfR(#72(2)SPxmp61~#WEawe8PCTXGf-!A zYj~Trang$pZ|7Ed>o%60OF_TKI=I1SBcbsrGZ*&G~!`s!`p>b-mdC?i~FADs8zYsG$)cezov(Dyh*#0;+k$X9w~o5-f~nl zyp2EL%!3@>E~@hO;+z`1NwuUMNq)u~wU#|lr{j%Pnn9=I&Dx#e?K?Z}`kuqv#Z}%` zKkC7omEY@5Yt&ka^S!)9pMc?Q#e$>U@84Qgd0TYlt?ql;C^kasN?b`fqffgbpFYj5 z3%%dgue%2SocQkletJojx6$)!@TS>?bdHS2%EO$J_N~=e-h8{|u6@@Q!&|8f4{ypexF0q&DxN?OZgekzTfDSI?;HzR`N4W&9xdWtw~pYg`&JL$G}F*{I5BF?+7J(Gl~2Jp8(CsD-#i*W^HXcq(vyxit1ENA zefFw*tbc#u9mie@-gXYUJ>7fCT~qVa7%x)ieLv>gvwqqc z6K<*ruuo1S`+no9Ky7%N|DGf87qrTM-rE_xjT?H0{ypU`LVAmpC!~jP-kM<-nh6xj z+mQded6Rv=T@K#5-{QfWW_PJQu}(Q3Z?UR}Oi9xNJ%#di#p)BkWAk$ToK;-jm{~Ic zU%!-7lP8-uDRYjv;qB6+x4g~1-@1agIV)@M#y?kS)%Tprcg2iTvo|SDPc!K9DR6hx z+yyl~)ZCG;pKF;@qcv+?-k6!)G8^Oc)5Ca48{U4t-$^TM-nxOe+g`508*^$tLG-s2 zo$r2YiPLa-OLRttw>RAN)mb)g-ND<`(RamtPg9&oik$g)i*-^db|m#yC~uFS^P=dj1^ib2I(Q$cWC)A}*%>8ym zk3XMa-)}v^Tj$ny$GzXA36MQ%$)7qNe!ppUq45;6Q&Q(!QMn7l+u#1uceu^lmEi5E zaRu?_yT{c2aNbgP$o$F^ul!`kwInU+t_^QTFCB|tSX2JJ&Q%e3<20zbYwB1Y z=JeI!lW&GyNV{flh4OaCm)_mYzTbL-w<+!J$$n4Sxn_6t@O{_h;aXomYcY?c%csCq zQbpfiVlL&8I+5bcLoNBGxXT-NEv?ECH}~7=!w+0~9qi)Sxz1i}F z-*2gR6v7yLb9v)eBWlCj7mm2^0sDTt3cPKY%UKy!8cd4|H7RzNa~AwTL^7C$8s>tBTaIPN7iV9-H&!&)B^618)l^*WfMI zR5jlx&_m50LMAWeY<}K2fe@X|n=~IyMQg3O-&VYS`EP9A`h&OeYZCIN>A6v!y}U)A zfZ=W0q>p~V=4}9Y>)rWY_dQL`K*)U+vX=YOPg_$_<6-q8b@u(nPBgp7@EG0}{Nn)p zLj3a2rvt&;3)5=wmXWjO_i4UsvxmnNy_P0GN|C0$x;csqF2;*Z() z+aU0E_xc*VY3fRzmJ-+BZ!KjRFL_)AXtFM4UrSR^##} zbsG{7nG);fcP175Hj;7YE4e_v6`4kw@WcKxIa;ElV z^A>SOhPU}=|NH=(x1r$e>F)Puzo(jh=%JP-DS0$;dO~i1@cT{E2|XdxU><5IC*v+} z%phrIpr)Vkk~X{@_=spM@7TiyZ`h0XKmhwgSVNz9>{)AbJVT*{du$HDTcRZ*ma@2J^$=~e&6P81b7>= zU=QKV_mGerBP^ z>aD_tZ`Cl1;PgMtStSu|sJZQoOX*@b=B2 zr~lmM?OO2mN-^pVahF)1%R`)-q4|6d2KTB6*mHTJlSAo+k51TFS$? zo;Oxu21&EqlE?7&(QbzwZ}T=Dysf`_jr*RmF11uQb(}p=bH9vB{m>JaH>si&XD5;; zR$R{;E9U6X@HTkUBcHT+n*iQ!UtWVZ)=iu2c+0Vu%9|c*ZFBM#Yj@1!@|J??Qbohth+Dt!{CkuW!P|ggYu)#h z-R`NnvAiWZ@tU7Xm$yV`YIr-W-$#CB-*1z^TiaV3z*~{imS)>n-lVQHao_0`%G)_# zenW4Yx5?n`(U%&)n`Wx1JuR0v-^pb2=1Xrzb#uQRb7GIRHg8kF+btv3?R`)84E7{) z((z{f<|xD4kN$7xr)}P*g13pcHxX~r#Q2%gc<7h3o-tOWmNIMNvGQnYW%H)Vp!68t zu4{k#7Mr(e;O)v+S{QHKyL4(&oNM*GaaBvy*}Q3H#a(Ck4R8H^H*lxT+jQ`@ZS?x$ z-&6isA~pAwS~ov$oD8+^WU_f(#_9X ztjXwk3#@2(`@h5QnQ8Mj6TB^J{ZR4uTj0KGe%3TO`S~q|H&)bin8H&iZx?;y=V#cw z%>r)|uH8d;^F7&Ocq2c)9dfOtH8~6A?co1>=YuwHv%%Yy_eSQ8Z#uO!F~;eqrYDw8 z$6H9}Tube{Yp!*9<5#838j_i(q|JN!lZ%hP+vaT!cx&H&L(6(kxkqVMns{S*lXfA+ zH9L|#E^m@EFm8A|;(zva-qY*B+cVcSfH%!&Cvfj|zsF0*o78!ucCF`)9a2j>vf{7) zys)vO^Y87>1#fpg&;Z`pC-+57Pb|&+o|Csi`<@S3z z54=sk^x>BEo(9gOa_^p3=W^TAvHDSLT4DV#lP@)fEz zpN^2Vn*4Ej%cyAXw+G&Lfb;t;7l5~2Yg#aGAt$A0ETl@t>3A_I<3X)-5GoC~u=Kym)_`w}s$s`SgUmHQR}hJ(o9rm7Pm* zdO~#S{Dtzi`fF#s&E{N;)+gQr(bvp}bvq^4n(Eye$TAy}CY< z?mgv{n(d_W@x~p+(y1ro@|LKg`FY`qFV1%Uy|x>{+e@=*@TU1LRGPN5R*EwZ=Ri%5 zlt=P&Ep_=6n2nlFjb?Qzj}({kYjR3!8FzW(SE<<{H9e9p`3-M9f4XhDeZMUMZ;x%P z!JCvbR-BWJm6Lfi`QzSiQbj4wZc}SLZ-Jc{-rC)E)L(4gmV&pNdpugqdm6ZxD1Umu z9*W_O6+?6gw}tZd@so$W*XHde@HS~~19%IWUdX9&ZCu`%x!H7~y!HL>*~@I+mVvjv zn;O8I?}?<~b1!evCt%*wukZi8g;n0x?!WIp-+AnD$4u-% zH>rvyGuP73I5j;QEv?nWUEZXMnrs;!!`q5?+%?GI?Sv|Ck38wYo7AT!F5NSlwXr!^{M4*BIlP@zWR(ZRB*puP!snn;{PxPN7Lh_eSp;4Xt*7fysGA?iYYD8^# z`^Klc;-9bI{ohZ|s`57X)(E_X^sM~s?M})}zs5tqM*HT>=8f}bW+|T0 zq2aCb(7B78`|a#1Z^O1Xn754n>oUgq_~vnW(_{##sL7wglg(SiiiWqp&OH9>4sYjF zdAoGvQ`zsShI8LjU`~w=$uG?_A8%4N#x*;lU!$d*Tq}88-sqGvFitJ4EqM%YGddhS zz~SxODsL~}UWYf|vu1D9vGOpdMoV?GdE+W-O-0G0$)oW|`K7qao8)A5YK_0-F}!`f z^}3rK-u_+X?Y>v)@Fvx2>GAR@Xr}7B(-b*8&~DfhPOAhJ@j`DZ|7Bcn>S{2 z_IsM6Zr$&@njvR8-kM=|?4eNJI!qkd-{I~2DsLn1uEQJOYijOD6W4e&IqC7eN!UaA z6qt=)Wn81VR?^Hv595*!@n`cEGDCJ->d@S86F&LisSa-!RCycN=4tnSxBATOm-0w)_ESCuM!1S`YI=M%^Juhhe#xJXH}0ZT-SGCexBdDy=YG4S%G-+7 zIeFtJi<-NZ;`D@Q=BJi&($hTm8@rM^m*SeW;k;?8lsYuL{cz>)+Jd)rum1h|*7$u6 zV>)edA5?oN~D z*Sf$-8MumC%FH|^ZGijP)=zb-0$1j(4S1V8wFYleEvYX3MQW)Z#wA@o1?j8Y1wA2g zuJzUIfm)NtmQkQ6VhPNL* z|MWEberpHbuDoKa`<^E1r}7pzLw-GP{Eng-I`{2?r0I9G}GtnN&E>$D5Rm@mL))C$(?>xV&Xl zH@tN_efe=VZ=J#0+Ie+&V}40<8vK5eTJkU+qNQ3=&UCy<6{R?*DtWj&YWk&kT;8NA zrA`cQw}1Y-zp#0`9K0=mybf;}`73t?Cq_RtC*!M`hdLxr`4pH<(!NzV4QVZFNgl?1 z{ajlpZ;vf|J^lxg@_%2r0=!+{=lSgSG-RjTH8pcKO1r#iI+Q$&OIq5gD6jtGn3uTX*pG=#6!F<9n^qQlHY= zSbiyw6fd8GZ&y;b6mh9HO`QE?^G2tXnI3BTOL4>7zmFdF0h_lT;BEbuI=nG|2HP#} zt}CAccSpVF?UdAUrxCT`?Z;Q#{x_Sqp5SfipcnRx_tck=Y~CWCLc>3aBrSZ`8FMzg zeWgd&B{pwYg12jz?IFDR5)wXn--@yPJaK9%e_YGCFJkg`dc=`U%++rJJPzMsuo zAMp0V&2@NVP9^6xnPi@3&@OMy&{c{aT;5VtDdjZ0J-+lU&i!^3czf{0I=qo>YRzYa z`;z>OQ**6GYu57fl8-l5q1JRrk7g}Bn#t48S}t#zY#I-%l&TxvPU>;bpX~dsFL+x$ z{H5&olxM{J)EW={&7iY+Yt)W>>uUTiZ=Am-1O1XNzO%u+X;{*DGUiZ|$+*0w=up~`Z>>Ukd*@fb{%)JMLE!C< zT@B_^`b}xtDZ@!c9{mF}d)?%FVVVr(?vUy_$u9C7zaVd`!XDyA^U0=V$FTVO-LzMGxcD z8jqegW~1gVGO8QiUToj}J2r14!P}0@cIw|#)@47wTIyNKLqD~~FL_+vB&Tm&)1fA( zZyxD%^t}02w02>5>-_PZ&iytDygfS2gE!`vc1KS$>vX&|+phJzX*$n1cf;FHZyEY! z`+ge@-fmu>kT-r_s5SeNJkoxdpIY*lPk~OUqU2$mniUymeko2*ES=4p)CuF<5w+E! z;qA{|wzaW&8w1{Eb=jqVPp$lwKXZ`c8NJnAE9Hz8&%WQ-xuj!tk~&W|Z@$$z4Qs98 z?X&;&<5O(j#)7v2vl8-V_0w?1*iSZZbW(HL)btdoH9f@TEn|0vw*e;~vdiXe9C+LH za2?(>JEDg=qqmyug{+loNjY8KBquX#JoGn%PRE-y4a3_XzS70{?+e$0x3%3{zaIZ8 zmU{}En3caKGxRq~Yx>OIsit#H6^%zzE5s9*H)%(uxZ&-%wlD8*-*4l=+k)#ocnjIJ z?^=Gp>bsVH=Aka10wdJS;H#O3T9ZdQO=&IjQ`7JAMkh5_Nm}xk;)b{1p7*EGHg6Na z+tkOJls9R=vEt$Pn{Ou?mNXuzuC&(WP0A1$H@qG8-d{U^KkGX1)~{C^_dS(5m*P^N z8b3YMAvvYBjK}3o(>be4nt3EGt(D@1^499p??2kU-zI{$7v|UCP3qh?&WZW1P2p!h zad}J8ZLFMy^0wwHCpy2scM^De@QH-H#hO@(JbQVIJ^}Nd{@|!phuHVqWbn55sVbD<d7A>>MlGtr8-E6&mU{5@&`&Mp zXI%12ajvD#$D5`KJ=CnuICU%!bGp2-NC29549#| zED!5a=i`kTT8b`)x6*kT-hS0-{P%3$rh&Ki1KPE$_cYci@8zvL0qJus#WUt_c$@v> zO=E1{rh~V~m(<{mw{b@P%6BZjekrH#9)10+>+&XLlj6*#@o-Ai%tK8NH9gc)5AOZO zY}Cw_LmS>UKG$o9&D#v{cGuP#yyd7>VK3HN&VkxDKi5*byz#3cRiw52jcU}Iy7W*> zwHSAKV|8j~W}KQHOB>!!+w|ByHg7Y*+uc{Uci&TfQmDC$5S=2YZ$BAn25%92h<@+I>Mb2_u{w;Er%>J|Y&dqI&D(77He_3a zdE;$B9c$WBo_xF&*`cPE##1P7M}6VtJ~nT2z+2m49o+X+I-@-)UOolQd1-bXE00tw zB)^`wkSe9l&3iik@cUQTyj>68Hr|quH-4WX#Do=KFc^^>aSHTF;wr=4SKf<1NQ73~w8D zK6sza+dS~La74$(zNbYp#u=7RAw~5RIoY|dE|j;qAJ~4E&D(tNHuClc^QM_ziWA(k z?>D}Nh4S|KWgFgT^R@uIb$ulQZ`>XEVVs(tMroI~5URPBJEe}bQz@qucX^|enjK1U zdZtUr@CyuI$%@4nu?-xh(ln_73;d%yjE_Rc%p&Z64? zh=`~^3km`XS1hO?_G@pbpkBN*+Z6#tA@mNRm-LWcNUx-K(nxQFR0t3tgb+#sflxzF zfKZa(9yZT;KhMlw?^*NCyzkySrM@C&9-;^ zhRxez@HTOL1>W+WfbYce@MoL6W-^jT6PG+86L5LUqe`hm!`q*Jbxsark;W}{|i&FM&M8JF_VBWd>It6kn$#W!1u-|*J@U+X_@^R^Vc zHE-2!=X;v%u1lGGtnY}Th?3cN|5#E?FvoKjq>v!Ar)Xy|I?UYH~S9Fnq>s(l+qz!MM|KiVn zu=znFTD4jo|W)6XF~-Mi4dI^)#zNSc00b1glLQ%h8E zEw!FER$&IIqUATd{c-<`r#ZZxpYXP1UPj(DJ;d^GU$M0Qe$(v2x2rsINdGm%lyj_&=_UbJ;@)qk88Y>S^vwRA9WM-8-v~SOa z^7ii+K6tak+rWUXd;lE=5M%bS#0iu+cS^5h*)=Plwc%&!*?x#d^*=cn)ec~;Yew|871u3%>XSb4O70Hu#oIRxTrs;&)=x4m74R6by?sScF zzg?Q}*6yVUys-yKYfjMDqw!1qr1K`Ns>gUZZ^~U8-e&&ilqL>umnFPC*T1X#o+|TK z{C$sFUQ0g9r=aPUtEe?=V|haA7Qy_r;7zkTdZ=SfI3$mz zA3bj&ooLo-x@8{8FU5=Dt<>SWynW&IV;tVDNO-&H)e5|EcYLEYamgd)pD^E7wG&{07G`y|e^OZk2yj_{_Hhgfm`g%{LPqDT4if=vFmQR5Z zO@>&Wkh;EWv+B$sy zsphShJd9II`_jZUd0gJOikcOrI6ac)T58G9IQ`W0P}Ae`#;-Jgnsc zC#}-NIZyga+5k6a-xil77hIXQ=HRXWy^Zk3&TBzidu(9dx3}^sXjZYJ4{%baGEx=p5Z54PU!&c7B=UZ!e^t?%(S8GbXb&KJx)Rp1wkc(!I zw0Ua@-ZqWtk^Y`a(`MXv0$f`y9nPC?=hd20F}#(gVR-9y{{Dk(-dcgTdsau_jgzm| zRM~?@r}I`XyU_GmC~v>|Bx|ApKum^k_60pr82}r=K2bdZ_84 zrpM)tPK{;v~Pv`&F8=tayYYX0nudOm~(zF@3_T}|!7Ty5V*?41D%*J>u z&HU8NFU9H6^QOs`au$ZSpDgM751Y5^!CSZ1z1;Vd-BwF;U(~++NNdA+V<%D-dNev# z9@f(H##I@#;qBo&SKn^))(*UFo>UNTc}z0HOd9USxV(jQlK0xUyrrC@;caf~y}oVp z)*ie)_;5kIgav}}1`*s_WGcIo_og3czy*v>=P__K$3mw4Qg6n(N=6jm5 zugs^$oaIvpIY+LgW))wp$)oYGZlS!LecA=KJj^*w@%>g`bRVJrkO@9d#cNuG?`c&GQZ0kzZy{+ z-uBqz`48BC%)5`{4Q^jGmkiTN1aElLV5f8+82(pdFu+^=09G6w>)-TYdNdSTanW!l(+j& z?J&aTts8h7*`;sX_f-0{GEQCOG&# zcktG8P6gh0F1~er{YBPdZ!T~AY93u_a%w#6Q=?tpG^;Xr3~zUx`>^%@QGTY$4dCtd zCoAxlp;q+ouk!98owtzcc`y^wPoca$a{9h6+4oxy@b*aee&O#a?`P_eDaBgL)2Ggl zH%;g4gnlVbkFPG2w@VLu`lmK;J;B?e`4M>Yoq*)$w5cV(Z=5-+%bRcKzJ7P7q|P^+ z(r@0=zums5m(5!*@HX<<8sbg4uPASKdHy_?BC8i!i=^fHDKor1bJVh1Y~Ff7ThptbImG6z4|scaVFli-JQeSg zl)4rDnw%O>th%{=(sP}FW=izL<&BvwZFswA%oW>g-ui;KyPwa2Y~uHfrfpLk}}s+T3rGzp(Y=HgEmG+qfl-@Mh&~xW}X(G(AgerCOReb5e6H`Kz~OS;sd;cdmE7xl4u8vx!0zMPRa)(x3P$Xclf zDJOq4O#1yMWne{W`l;!umQLp_Vz-93Pd zku?3(lHWI;&YP4?>O_i5c^KzfPRv&`5A&z?i{hPSWYcj!AdZ-c?x)0;E$mZ6`juqXAI zjW;P@+>w#DoO&+o4s?0bOup83t?9?*O|#Ps9>d$gXMJFQo429ht@j%l zdCSm`7fYH+NjXb!@%2{!v@EFPo?i>r8rMVGkNBr zW*$k?Up@s!G}W1zYbDK`^hkL$ae5@pTIsyeDP>c}&HXlR_jCTA&D(JBwrOQX-Z)8R zKVJ7+-Z|6nx4f$tnLnL3o+EXJ4h?U=9du|{o3|0*?Vh(9;f?%Lvp2>i&Hg11SU*zEfh) zoB%aFlBS;aX~X^3beB%~~#Rl9Q_>j}$MhHN5@f`v>7a&?^7@ zkWt`m=Bh?`V-Gu$cJH@(+@0n&aCysVr-rxI-`usieZP$cZ-cf+=1nsJenzOJwe)aS zl3$8*ZFPB*cEk=tG$#<3H+D{4>eldf@jp)8Zu2$g@0-8FaWyS7Mw`4nPR z^xd_uKcvsNyrpzvc>DS}f1hphHWs`+cYj9S0{?8c;;&V$&r5^HH=Z(+hMaMEV~3oK zrW3BEme$fkO+R}G=S`|&#SL%&cip)U+q{hfZyQ?-)!%QiK9#ZZaKakR`IJwA5o%_W z;`C6{PhG8DyS%Y0b}ni9sV%?Z?b74+f6?Y`Ja}7mlLv2B{)W4PJ#eawOKTaYpPHUn zT7SQVbi!%G>YRC`-WYdzV}^*@@b=(wM?3%D#{}@U;6V@GxQmE=R^*nos99Hv(<5p3 zexp;$>>FoJYAKJ#BQY$+^}I=4NpW^t(uTKt?m6&I`+mC#yiIO1O#hx*`5QP1iAz&r zPK`(MOKbJ_8#7BW-yxlM*BX4A$TI!h|YK>p=_{Le+ zz2Ep%U(HUqR*KUjX=yFv8twANip(IzC6DDdyuEVh@4jsFHVM3SU6+wJ%_myp;a-Z= zaqqVxJNK>S^5&aC={LOf8`$0X_gRy{+iUHH>)%s}A1SWv)9YQ1RVzQWs0sVt8Z4yxEiTySzzGcB|1=9>d$!`@Ha${r%g`;B9E95&HL3(?czL zIG^HqD{?aKPD!1n6K2r(OCH19(VLzaZu2$`ytSG2KH`mY+c9|4P(jji-L>Iu(9k`; zYV$T7ygm0sj=X86K?aJ{<;mA_hq0!i=Phs=hPQA3!CXJ- zHTaz>spA|XYQx*_rgS*M=4~c;n?5&3-U2@_6;CZz&i6BKv1VwoYIqylW!pfTw^`t= z|I<10X7y8%pCUb+^ZS{%JZ4^+t>Nt(qYmj{^EMm2HSaNM$9_-A80Wm>@s_^Rg5J|! z-+cA}o3}aO?a2imyh%IaIo6A=P2QvlOL5J#Nm_opE_G<`x7(Kdb#I%ux!~=#=RA1R z?5eldf?7iKtv3Z*Z-frqWI=}bScQVq1^N8oioA0hA zzZCbKMmF9uR5!eRsHgk??al{pT^HrZo9|?b^vC5b!|CMut0(r8jkgSEVR)On%X25% z_uB&S_Sy?Oc{?jEdl^nDSN4kGtwgxtZA#N$t+RQ%1-w1dZ%lsgsWf?h`Xo=@ahEZv zD=D6Lhq3a9^QN4D;q9&6-x_T5wh+86Tv~xQ=BJJ|fgQZ(J3k^Vo%^pLE_d z8I&Hw+fm2e@^hQF#o%q*Z8geU-ltPO1>Y*Ze&tCue6sTJ7s}hp=U9rx5+_^37l9{r1^s z_i}!Hx(vKcxU&LpzB`J=K#_UMr||ySDX9yYbIQcb{nonWvd8TEZ8>=BzBMCnDRsR* z(XsMyhMJt&?l)Gc7fsS~-L>Iu+fl@aEftub+FT_RZt# zkIS1>UD^@j)KZ?1xXW8ewv@Gox9?qhHvaqi_jo%a;ce9D3E|IaN)KM&%H@$qI!%{1 z$(cu2dE}J(Pv=cqm7zn!+pq8J@+;?lJ2T;}{e2mE^Sy_C_r?5@=1k~eT&i0>1+JoI z25QN}xTKkfTH~jOYc)EZH%(@Zr__nz?K|5pYVYuNR>Ir%?G<=qernC`G=9m$I5q1s zPVMV)?>AK_~-TS{eH;#32ze~s=ym} zq3MDCdeJU#>^zTM=aG{=OWNg4ax$KwTf^J(b$f1c?zamP-nzA&RQvB~hP^l3!_r!p zH_6FUkaWme=42jfdL%9R^}I5JD@b>Y;_CCqs?ZSk&7pK%HZ`NKK_=y#FdAqbv z)96)VbsIv0o;OXmDV-SJR_?xVsKeVu32*nV%g9?w-I_f$&eP?MUzK#ou0z&J{b=HG zc}uBic-wZtLDL-GE>3ux-hOgzzNgZ@Vx3wx-lVP=kJTY_hUjd(W!Qz`t;2zB@#9qA z`}M*l32*(U*BNgiljqv}cw-f6>sNCXUjcfkna5Y_c?+y)czgPt^Ts>(ThoNMHX9=I z#?J*cr^L8skMuK7h<16)qe@7JoD8*ZPCaiS6DW0PczgKDKjN?VzIVS}n((%%(-ikT zWnD>g*U~f?r{>xa&7AC$Ii+~{6qtdle6^H^aj6qct&p{P-q@|CE6Zbed-A}&UUu%c z%M#vJ&#J(im8as(U+*6Geyi4wLRixDtmjRWEyZJad*b+`+&{0sJmGE5#wzfZ_k>es z((u_ShPT?;DXGh;E5qA|ul`IQ=YG2);qAt5Q{DGeb6&AL+&guw{8FA`c$2CZ87FDE z?%MEn!tf6s?C^GF!dshpRpzb8$+K3jlW(|1S+QEWP~N6KxaACox2qD~UU<67yoKCp z(po9b6VHz~O?7rcEv*fS7sFerLvz1<>+Aou!Qt)dgtt3-++5xFl+!L!mrp@D$09q? z)MB61nze=Uw*91TBOTtZNqC!hO9kGzBWlg==;5!gseL^mJxI0E?>9}iQdd%3>V$D> z_8@8cCGGOYY|^R>adW>td;53r<5b`K{kLlq-Ue){z*~k|HT4|}`A)4)NeE1uwQ+$8&RZT8tCcewZy6Xcyj}IumsZ)lwE%B7^`D;ppk}DsCNd(-E#xeow^;o6POnhj&inquzp;611>Uy4 zvXi%h{=X0MmiJ#?XRWxrF|$R!x!-Pgrsv}CH%(U| z9*Kmwyrpz#cst#_7j&kW~P3RxB18H z@)4W2PT;NiJ2k``rzdG^kB$Dm2mRFLQ(z=Sv)hn7A#0@`is7x)`MbX#a?FMaHgBE5 z+lJAz-_P%<=6t1d)U2f^?>|#uE$Tvfd*Ia<&$fB%0^XM1SAjR*3H$mvjcV!gDM+15 z@oIG&(ocT8SzVd?ZRFO8vu)nGg11r4=Ip%R$|oYRWKA{m3ohl)JDwkJR_BJdQP)qH zVe{4vymcDy!CSpgp7U|`_hB%w0Y|Z-ex@H!JDS8)Tb2p&B;8H*5oXof^P=;r79W^*V4~Bidy5R zhdP}%R_7{edZ;ar;q8>IzdF$7trvJ3-*%q+o=Q7q{(9H?`;F6}u9pe8yydhD!&|$R zUwYc+tv7h3@|M#s3~zIP(Gox4vi$3XKH%-ubsoG) zcj>kGQS#K%zI@}-X|f+`O@5a*O$J|&#;@sC$|J=&F_$-Km8O#vkKt`<|6O|8_gi1^ z_GJ6{?t7Y&Gs-8f7V^8i71Li5qg{Eknb{`X?pZ)JoNi&mp9)G8h^+xxRzQ|m!5RqLNZ%x z&HeW384o&tUOx!DJ+v_+Z`_epx3D*nJ@{%pZy~$WtPSyK)~563JGT_S;ce(gwobC| zx540TUbkDay{DR+TmtJ7g_tdZ;y1lKfno&Ra+)tWwg3x4-=1 zfR#3HH-fj3^D^?roz{Z({oAwp`>hss7t%vIZy}kjwT8Ee$9<}q&D#+0*5R2(c+>FD zx*EUrxoCKQrL~fub$xaD6qwmJ1N~Cx^hjFjL5fRx7-v6v-nfdInNzgk?Qb{sy4L1x zD0q9N$HHvyX-eG&ernmX^L~pvffApFx23~>ivIw${O4K2z}wneD)2_OLq26(E9Im| z!)J(}Yu)>eUzIwMJlq{M^GIvyp_cN{Lrp(xx%V5L%%IWCZ21ju@7!>x^YdzkgSTay zD)46Ismgs7G80Yz?x#ENikhzS@WkaUr9;Ep58H0Oz3Pj znw<1dGr!9lozz^#I6GvVnrkKP>zDE{&YUi9l9TZa+VD1Xw?%Vp-bRAAA&WhDeJ;-a@Ko1oj2dzr7&Q4yXPmT9dGkC3cR&>sRD2ObWr>DCi$6D(p($j;acV_ zp8~&1O{b*k*Jvq^6lYG2NAkP8NlsRwhngPhlE?6N%oUe^$mVS{cxydiQTThxy7i)q zx!R!qG_e0%-(Nj%oCdY;?zom( zlZSq4&DwO{G}$zsQYVJDu|qz1uFcz6@V4yL3cN}8C*#!IwQpRxm+0)J^Cs+t?Kmcw?U$ozipkdrYe3^2W}kRhl^SXms8=UEVYmC6BbD zlHc&Q=UoRqY4bJ#ybXCh0&h}pwH$YOlkN&>7xYt0ekmTZR?5kET;7=3(uTK-7Cv^L z&D%}jt<|t4?t7Z|1gt(A?zAH7y1YqU6&a5;0p_#t8@7yiEjePu^95 zx0IX>Y}C7_HhIfqs^PpTyE43W+2@_zY~Ch;x4Yh~z+1{2otZZ&C*#U}dG)~j)Oy~O zpYDoXv7$zE0`zO*l1I;*rXoGmrRs*aM}GO~Gi=@_gSTZPm%8sMJKs^Y%UcLnv8I$q z4|?7LcVT$jb>-g%*}P2wZxil~%$sIn^iXT|>+6?ty1YrwSn*h$`}R{PZ@;L;p}cipxp|y@zugSpwv1V(e@}hSm+!kq`%YELsmW6e zZy_COcA@dOys;}y2BpWmr)^&Oc^jLzY2a<+Y7gF|y1pDrd5Vk|!&@FZ@}0KJo79zW zW-E{3ZI_X~PPTcQ4&H8SzFdF5F@Ioh6@N9y{CVr*d8^jM^t=Vm-0(JR?F$`j-e!Qe zi4#0{<2_6*^+^x4@3u|N5EW zZNyp+-lR{c6i?}G2fp8`H4Uk*p0~jBGQ92g#uu-&@3&dttzGNevc0E)z3srfaWWDE zd3czUx=`L${b|JkHgB`R+Y6Jb%o~3mCN(D}Y2N>mM~atELHeq1oSlbs#XQvPhxw`L zp>}y=W+?+b)XYp>%46=gKTLdNs?FOR@V4pUD)Yvki_~#>E3$LnS}t$C8I*p*+t@#h zJ<{fFE_hpY{q5C#Pc<)&a=%`0^iJN=Cy?P>3~zrv`Zee8lg|Thvu@7FTZVor@(?SJ zFO%uh_RUtUU1;it-*2o!ZB5DWcE;VuzF^;P^TAucM>Fzf)vCx6J<|6H^icc$yd$pF z=(xOTD$>JgP}37jr}M_@)U2Mu7|-4C_K}&F{L|)b0eEZGaYc3CQ|_xsT|NbAl2Y8a z!&rHI>(Zaj8=ZmL@HY3mFYRsfb_;lWX2$!Px4;SW)16u8&19jx_3b(68Jo9-;O*|m zbL5Saq>gpUQcfw(y6N{DotkX)*P1Srx8{Fa@|Mlpt>A5Pmpj7W(|SCS@+suaMlDPx z8*iGyRc7??_{#wZ<^1n>>5DdROTpX1`4xC8a@xL=lzxwhJ!?Gmd%tm~n(66z(`4gp zsY_j%`>k)M78`8dmVvji&sLc?PEXRBIY?`z_>RGwv^&Q2ym3_qZFpOL{gj{Dye$WB zU3=Z-zNZ;#ReVEhIZrxoc@XY9jaWSt%G>&(`0F9Nlz%??ZQyOo!V0`u_hbW0nz&|4 zQXc6<=+|iGk^JRTkiII#rOcYR^4cFJ_uKm2c0G0P zCa0Y;sr!m9GfpUR^y>2!aFbGM;u_qfJRxhPoKn2@uQ#c4-zmAcNtqd!sz~wDT7%o8 z8z07hf4WPVx6>2e?(4gUm>zW@en3 z887(_Z=d`Au4^3L&PsUeJn-&p@2OR*uzM_4-E`hK?RwXG-U4@OcpG=%`5$w5J3HZR z^YTV`BhS=QZ<0rfOL?TYZyw2CJ_YHkQgtaFE02_)aZN3H!g*u1luit9t$y*Hz7B8a zB)mPnxe?x&KbF??FL^XQa4qjc#^c^^%uKE6mL9I9rpH&iyfL$+eXErGhPN*?JNHtD zw{sKT9=!3MZ0~7?elowgWF}VKxV**cDvvxaZ&H=OIUC+i|G=@=IlP^h@OH}`8F`cb zNh0IypK)qw@6zXvYpJC?Qe4j)t58dsHF4&l=B}AXiqpe9)Oy~SjapN^S z$MCl8!MBz=yj_s+Hf#93?t2Q7uB_G9Rv5!W)mq$R)08#AyXbuWjvOA_8TuFA+8Yq1Av z$-{VvmTGBoR+qPs&ZXT+IkWL5Ws|Cx;)b_vfBu5|`)^GX-WG3+h zQl9GaCUuoZyindg_m|D?uNN*&cpEkDKK*-Yz1NMhq?sObQft;B2GmndwrZa8nuMxFPReDfE|+n^1fSm1pBc6q|v z8!c96yWjYUmNZX+TJkeaKegmxT++-_J_U`Ft9<=j8=_g4c{K5CyfL#zm%1|d+p;E4 zJn!&!MZ()_H+k^Joockyr)F*5p8W2&ygM)ce$!MeowecZ8{sYF z_Z;$`yl;Nat$um)-F3G6&9_R)Z+Kg^X`1`%)2kBR7Pq-y|DO8x!1JZ{&ExCW)Jo?~ zlfk#EYWcJArkQ}{F}!s-km>zC^4c?+o+vR3L` zit{@k)=lS)PARi8Zg~4d|9^hRx!axYQXX07rm@qWyEDA)*6J^v zz?=Db4tqCghM(s!;jzfTaWlSSlKh&uj zxNTQIZv8yfy_+-#Z|yogkp7@rIip-O8TJyTF87j+w^*IW%EKK=Iva0Nwv_n0-@rZC z_lL;^SH9m`fVbyoHo_Z!(>A466dp9SB#-Y2mQR7%n1OL=*S>x!j}({kYkFhOblzBn zni-_H)rsM4%{5&Y+q|^|Z}&gZ2yd*rBk64S8z&&0f)wXkUmcP&oj2c3Qv8OuXaBY8 zGd6Fnz}xJuYt!FTi{XZQjB(#-aBWKe4Yl&{xV-sxCHbYev zc*$dUJM{R8_yt4xJ#7u%Cd_SwH`WT#(!Qje^i%t?E#-H4lbljKq!Z1KLh`d`mp6V@ z(yUHRKX*}z8{U39@~lDj{niG&-SBh;-lY7jRqr~Sw_2Eh=A^RmmSJ~>x4Zv2>{~W( zZNb~t8y%rTL3o7u&9;n#^H9ghRQa_ASXX8!U9WzTF#(7eV=b>HRq$<)b z7?*S@r{V3>zh1eg&09P0_SkbCyh(lf#-*O6Q=p%<7}sd7EuVt#DrTUT^3Wq`X{{9Z z&ExXM%$jWUq^vc(ee(L1lWgAFgSSU|Kjglr%r9xyrKX?!&_hkXuVx-^^qKv$;2aV?RH2%Cjc$ zqS;X_Pe`BXyoF@8)*9Zve9m3x*t~TFZ!=!3z#FGTZRL;teSVSoUEb<_r&9lg^7e<8 zzv*W4)(O1z>;G{0dm6%tbthM3Sem3}t<;;#o8*+LOa54K-`z=l7Q@?n@9D7L{Li&E zZ=Jzg+hvXLCha2DR73KF^p=gckcuHY4Ov?ZZ>7nY_w>5vt-o#a)&;yh^GZhEd?zpY zrBAWs_no|yN6(vYMPENBAgz^lA^924#+&pvotW9uhPM&NpLm4LTUYRQ`=CeC-_wwt zTK!afD+ zpO8FxFdvtRVTJCfq`aMz4eYx2+^OS5h^-ZY)C ziln7HzJ5)fY`keQv)hv2@YZSNh4=wU<-h;d6TH2))q^*w2gcb0H9gd^Jj|)lRmBvEF@8rJPb+bEoC8R?4Y~=Xbwp zs%tzUQ)T{i-slX}hPRh?`S4LTZ@t0W>U$#a7T9OS@A2};$rB9cjoI?(Hdf9;dHe9Z zGsoJz^#N~l-l@PFCrK?$fcGu6t-&5adz`W$Xf^xd7oxB-lQE#ao=6U%EP*P-U6$e`|W!h zJ9f8u8wlQJKH$L{`_a7Vl81SyIVG;uXs*@h@+nBwH5oJ>P3P=^e#WI*^h+K+Z_FT7 zXGKdJ-ag%IgY)wo27$Lxtu}=Z)-E*NYCM{Lvhk+LYfitV7@N1D;BD5_jJ$F3{JxPrOL2NMnt51PvzC75N#~7DO%-}H zYw4l(%|pMW3+3&+v8Vsc=4}{w8@WCsZ`>Wvl<`{D>Acli=e`rBzfj&jIBfFYZQh21 zxAq;LDE>W-b();2Z_fJS&3AY7=f|5hfAgN6{C2ySZQe$Jw-={-@Mh(2_})p1bJxB) zkDTmb$Ky?!Sbn^5o+WK~d+O&Oa{m3oNbt65g9mR`{s!JOacLTwwZ0xrEtfaVDorQ8 zd3^m+4^keNH_0hwR>lo)gI~Jw`}X}d3cOA2ys!Lv@65}mz=$+ejYnEr3vrjX zoVqf+eXHYC=ik?l25$pqSKy5krp~EPuX|F{gN9Z2e&bH5H5KWnme$gv(V9HmHFN5D z<0@)pb;H}xLw|m=eZP$XZ(X0Pz?+mmB+hBX$|L2}#9iKeyQ;N+rOW;~0X=WbkU<;X zjvuh_7dCHW!P`sSp3L^1az`0@t9Ym7kyGl;<*mr-QdfDz(|O~rsWWt9cx!RyNl)3l zjRSA@&dbP~)K6f~6**+BSovMvScN)PMdmD0r}GxE6T{n2e?0rkHgDs>+l*%-^A@rH zM%fFgrROcAinKO_Vc)fS-h4A#yED8Ue&R`OZQdq;xBfk!%J!aGwQ^-6k6PtZ2$^um z+B|k7g?tIgX@;BEUYHN>0d*F-#J`l+SKGw$ARz8QS|?3|rY`{v|YN$Yu& zGDvZ&TXVm?@$uVd*}P2zZ<{tX!kcDFQa@5WkDO91Dem$nIg5;kOh#IpjW;P<$}|jb zPcOXvOq;h!;B94}r``9I+)A2t>0z9j9%_0dO~0gl*OpI#*_c6!(?d-^H9fwXd8nC( zRM116jW<@2G^>=f;q5zzul>Ev+hp*zVsQoDr2HXqP9`Le?^>yD`28kj_U$llzssAZ zbB)L9-0-&IsQ>p9o3|<8ZNf_xcq6yWsfp9$t2MRg3DN2I8#8kkjK|WRzYV9_yiEmfBL+OYw3|Rb5heUeFh~z*Shzc zw2EpAOzO4t~~sPnm~3R7*?!NO4V{dfqhIG@g)dHTfk^Hr|*`*}36u z)vyn}Xy0!$z}xZ_Ir66Lv*CL}?vYxPN8@46;(25BB6T=#%5DvB51sh^{x)wj!P~gk zEAXbw-zX>5%A;X#CvVXwV0c@z;QNQ#yv+h{-G)8qzNdL`qG3Q&H&&kTX~*h>yUrtL zq4(Qq-@p4Vo448EZOdH+@#Z^K&Ez#6)}>D8EhICmglJ6fKKujeh~ zbTrIx7pz6C$s_q)-Za@X9(G>x8{P)}<3;E9L*|3G*4uLA&FaVN?h09}nU$V5O}1M0 zxV+^wTf^J;zWep&_Wia1ygfVSdHs9JKB=Wo7EeL)NO9(89*t&BNvHG1RZ>NbhimDl zrbn}Du9Y-%y1a1}os3g!x+>){ynXJa!5^}Dy9K;GyxM~|_EzsYows`59s75A<5weU z!`tJ7CwtY~F4KZ{sG^F>fI! zSUhied}=f~(|O|zshL@d(^JysemkK1;|JQjEdp;{*48m^)_%QihMl}cpMc?Q{kN7m z@9ARj_FAhK-S%JW`)=KiyIl-(82~k=DArNlxExEx+OIqs{ufV85qJ!Q0G-GxBED zs>r0|$!Wh{J!Ipp7CM)vD#hdSmNEgu+XI&@bbem+GVs>_`j_1Il%1#K_d4HLdDvTs z&c<6v#cHi}dGqZ=b9N<<;q8LIef4kl{k9yuZJXK%Z@#_Ni+?9?=@ZZ}W}UU+?X+Js znPBsF8+hBeekX5drBB|PYQwG06OpvEw#az+{if{J@b>)4hs;ZOTff_`r+&7{X{Su; zzM{*F6H1f2y~E4t&#AIh!y!n19*E>AGM){wB0G##t5Dol+%)sn1h-wvyq%u#Hfcs9 zyz!6GsAEloIjN=mAs((Rp8_LN#k}L}L8CSOFptX{zba|pDkZKn#)EbZEm)2(Ejk}B%bRZo?k+_e-mco-|6%7n zJv-rT`;!@YOQ~Ctv0C%HyybKXUem~<2bZ^;&cg6^=iSG<@98-SZyUS68uvZbOpG4t zoaWGQ;;iNJ#_Du3UM(HckIS2sjd5z_t_^ROZS3F6x!=xBcw02T0&o02P^y)opN4an zax%^yB(3RzYneZtH#%8`*`zqv`fBFXtYtrv-{nnmYT{DGQrz%%{52o!?(lYA!rP>0 zEAXbNTkjq{Z#%|LNuBSGl%E2_+n2txdVs^*`3Y~mdu@*Up8D>kTK?MMO?jGLuLGw_ z(sJFk;qCh2hpusWyCC6h+rm2LO*$zl?w$f8wX8KelKL-{w^^IN_L{@ng$ZvPp7-F5 zyQY?QS_^Th&s;z0neU=JCg$=c%_O9HDW|#LUijLOF%EARCA=-}yJhElx?{8F@)p7s z*IL{f-ZtF&%pneM7bm=pTT+2HD^K)Kt8e~1{Barc?U1J?=~y{k-uP=wYN-<`ZgpsQ zJMy^O#yh-SlJM5)`89E?hs>A~fV8Di;DCx*Abp7oUv zJG?bbczbccYjNLG%>-iYF(gkoZz0vCwRucW&s$)1!`nNXPnqKIc4@-fs^yV+lO`D} z9ypVR`?fZD|`Mk_lpyq$Vlzb72tE=zcuxj7?m${rfdSF^7oJ+;qUtZ9(6 zvJN@Bl(mMpo|hkSmBZWR32y^#d_C@a8f!{<=XpQ#7HhT^dxp1-O)ma#hqo&d-dfyI zfj9o`K5F(E^6MJrDN?iNxV$m5W=9%-ERTD?X*!fVDP0-f{`B^lCp)}dneevd^$NU6 zwRWUcocT~goKT z$6H8;A!|7S=5cxBSE-pn()3f8{D!x){@w~d9_hWm-+Oh!+np;tc=KgL>X~t?=Z4Q0 zJ=C#!2OS?0?9dl*7gPiyDnuND$Z+Y;hshfvK^F!Z# z>xT2DsTgvaA#2%NI&X9aYQx*6!H>3bc)K>?t?#I<;qNK?j5Q_Z)M(bFme!U}LE~fw zUq9F8p*8*Jd1E$eW#@*sZ-4G5pLck>F5&HsdmG^`L*2sO#E{;Kx!XdE`a9b~brCS1AnYZTPZO+&?<36Y<{WN;(8Gv2s&YH2=_XUE`8+BM@YZ_-SpIJ411UAhMhZ@o|7 z<20MMmf)>Vi?_1fZ@g#Nk4A^AmGb-M%*LBJl152;jJ_Q!`q#I*><1JTWj$4$l6AD<0NBE zfH^f4cIcuLcX&1h`*7K&xOpm1vZ@>TcE`4p@+JLvYt>1RvQ|6I&UGMQ?c+*Uh z9%^aA8V}d%dE+XLW+x?Wcsq945!U~o_1PwE!P|hz6?l_sF~8&qiRYc4y}7*cyI(p( zbgUhP^rq*HRj8#2SRTXMS$7Zog?+zW58m25;=!9#x5#*`6LfhC*#*0z4#_F4_06Bo zn{PID#kD1Ec)S1K$L(+P)(*TqdHp-?d&-=7X!e;$eofA}ylE=ZQ{)8Fd1F_B+VJ+= zs(z=~ytM~!ciim38}CnQsZV;SC4b&=_E25kq@A)u$s@(NmO3tP%xraHc>Ct}4qa;V z)&acDd(?wBD}UxcFQ|n)*?24RUeZiS(@$L9Qo1s{J+#M5|FL=N2;PQv+?M{Hrtn;K z-Zaye=2Jce?v9$PScO{hbFHstern%5^y_)^t!Pca@b=FuKm0kHw@%=#-ONUKW9NBj z-#y0aq4@hvvr~=7cLI9ed^20!8s7FB^W$@E-a3P~O^;WZH{U(R@@L~MR`opcSf}6c z`sBx(wF|@B-n(4#Rhzdi;B8fx?dk8SvTuqe zjk~krhPTa&W_`ovtt)t&GN%#Vm_L@b`fOm8`86K;eYKvqSe?YmlWnJ@&NqY7Z+Ppl z@`WGRymbR_J)g>vH%?g6%Dq%%GGwjG8#@fioX6Ub-bk9e>yU1(t_*KedoBHt&0BZy z_FDI5yW_u&3Gc`1!HbiSwZ3x;->Fow$X!T1FkUEc$1FYg%QkN}fVZ{tEAS>wOp0@N z^`i6Rt=1-xA8*!9&HXm!f=e9UdVsh2&sN~g%2V-pJ`ct@6K@=>^`p-s-_ywI&ughl+P^@x1x&g43q<%~L3Ezx{5r zW%m8n8@z3K-h(&aoz}9y_IdN2p2TE0Z&GHfLvz3VW}lV6wR!6U-X84RBHMe)x0`jP zI6ac~UF+*-U6(g08{^c>rqNt0X-yvK^qG@+vhl`jQbkrN`3-MpKD#gegTV58+84at zu_PmJtVPW}sp+YfWW83qU0(04R62ec&793W%_})`7h_l z8+)dfJ}=yhMl&b1X085yW9C?z9da5{oOS7EoSJ#Ud1K}jZFrmbn~C4H-_!o!ZPLJ& z+1}HXx)r}auQk8RTOJ*1*vO+UbEfk~XJEI6w}W3fqovK;0PxoHHV@v|4?jWF>|cs& zJgh~3h-OYbZ(POdQk))*)=Wv0haTqC^TrI4mUdl=8{Q7O>~Q=5xbpXJ1Hs!{n;YRx z$}ja(t+;QWdfvDr-zr>Nq-LMR@K)NL;q4=jT{_3U-v)uVM{aDD{+{xa!ih1i(Og@k zW^b(P@+LWbW=qkAw;%n_VQ1RB4FPX`hqq3DPgC-Hv7|XQjVD$=dft317P$*eZ-w%9 z+JW~te_lTnyuGus0&lf8G0jwcJ$l~ieHw-G)_d&qf7wboc-u0zt^Pfgb|l5wr{?zzC65%B@@x7@=ZzJa zjd7_u_Ra>R;oL*2apn2bVXg3Rlr1@h17X)>ktpwI+|AH{Z-w zb;H~K2VHoW&D#X<_S)nMyh)R^@@Ky5r2JBxbu}LPC9UVpx1z6K>M&MZ&s$(c!`qjB z_4@zWyxj!e9(bezZ>%d#Pl|J`q`6k3r92_=@+q)-NVZscq#i=@7sFerL&Mv!xtqq= zyiEje)7rI9e@{bpDow8z;`;kd(`_FA1+}KG#uLt)k^#fpFMro=q|MtT@YZ))BfRlV zr`GJ09`>xsLqGFKnmMWIXU@31u?nkew6vDp(&MZ3yh+_k73nW|3~xsdJaU}P+hp+8 z{;@`QlWJ8f9`_Wu3udOKCl8%>Ej@36yEDA)*ZcIVZQiDUx92-{NPkZ)hAaLWA(n^x z^*sglz_rxO@7{0BKqoallBQqMe1F-YCa1>3y6LAZ2*fz=IfzaI7FJ#F4@25+}Mv6Hv6+`EpaQ14pL zTi{L&Z-3wG6JN1;n+D!SckP(|o(A?%_4`oz%>Bu_}npgBUh5nk2kBL;jP=ApLYH|;0*A#`Kc=NX7&6&GKw)Tcj;qNJb{i5NAen~Tr|LRyS({kko?>=C&M_` zGES{otMTZ0V^Yb?ebnK829ZdDbC$@?M)CZ&v4q zx1LYVebDA@9(a4_g~+^xoI;Unv+*W%D8;Q4Y4rBaBTu2cee1fv{@CViK6qQ#uZ#Ph zTK8n5Q}UfE{YBPd-E6!uLoA)g1ei0NH)c!OwYlH^{OJA8|9`pwyscPTfwz>Lxw5CJ z>w5!8v#K55G`kM*r1R!G&y-FKZ$tk){C@j>y9K;Wex(Y$Y4)h`r0l)nb5RU$vAWgl zNaNwI(|KbyX&1~!&9xCVMQC)1_7J|3FgSxu!sgH?n`RX=NSb-5C4b&=J#T@X7~Y5YI?SVoaNan#lnxDVm%sS(F*a|D!P{L!y4B`;nzFBk z_dVqHD4#+|22F=~do=xr^QPQ|;jQJL-u{Kn+Y<0L<<1DaX?87rMx;3RsPSleV9vPr zo7ABcXD1qu@9vmCoHwaMt1H9Xx1RmMr8aL%!P~&C5qRV7s->;HH{5|zKXG}Js`$pG zT}W}}mptLTu}X?IydC}23C_P?SO(sjkLa#{PbJP%dT8|Axi;_otmX2?>eO7N(OgT- zwTzP!#_12yh4S{LtyegIpL{uZd+F{9ym8mm+%>1fc&%xvPnWm6JLe>+r4H!{>8()S z-dNP^DEmFV4ZJ=0b_L!-c2~=5d9uau#){HpG#+U!<5HeNd3$2_HJ2p3t>10eQ}=0d z+9{K|ujn%4gwk6#Z}biN=aifC)zUD0^U%+n)bvPNkDHWPic1xJ{mc`xwou%jYbZ*M#`J#VZ+&7Df|lE?72+mF`n>hN|}!rR(4 zjqt`=MQZMgTIzulmOPBRyz#5m(!RUU^r`1flP$$#c-!x;Z(rc>c6P$s!j?V5-_w*H z8h-mt!;B_RERUYIJQ&dIBG$x;;jPrI;ce2V{(QN^+c^nu<0dx3n`YXa3^k{!$wR+J zYjR5cxV%YD#;G+OYCPNp^HWQC=%J?H<&92iuF9YdZy&$u;Ga0WotyC1@1aI`)70g} zG=7aIB&TMr%bTVvUyr5}-<*Ql=o;4o&H9A(#9gjC@df9l(FlWQtgjd$Mzdk)b;q8$r8F`a-m!Z#!tV;Q%xNqGZ zgE!5t^}K1aN%Jqo4R1aBUV{JL`Mvw?f`qpP>l)!L;`<(jt62TGylFD?L^S@C(}^+- zO@2LZnrxf^{iSIb-u|cCk%u|ozg?K{Hnx55Z0{*+Nm}Yn@)sF*dGoDa%e&)VT;4)D zPg!etyZ>MRHQM3rqJ+23({tpFyRJ1I&RebRi2aZXmp3{iYQx*!6OP)?;qBstx6P03 zXYAl>igN@r_y(=6$Oz{}rcKhhJ@Z-tf`~A1e6W(5(TY)!C50XdPixg)b>Xdyo{LYo+m*ULn^2QG7 zl(eP_*GhiI*#qOuBWcN>e!p>*q*;+3#!K4p_VMp88sNOAS0ubW^>hW^tUL|pE5+G^ zqB@h(kkD0X{Uy_>AyJZ(++P}CcLff(a(KP zeW#>31*w0j7UR@XEqbI{jJv!^8GPfcLLDm)b857nH)*y~+^TMPyX=#nJ)Oy|mcV~F}(OzFZ-r?=)gtrNsJb2?NNII}* zFMc$&*eCtzywRD5mb#K^`TCisP~HYi{mBasZ`UNeb?MXJeNWk~uMU}vR4b31>AdAp zQQBSJ@o?Uh(=hkj)Y}$Kba=Zq;q8@0k$DT5o^oGN?!a{3LUtjoEpm74LC;%YhlaOD zfA`_X9Nw-=cw6&g4e`b)<*l>v7I-3FpOlbVT&w4eGiL_IQ?%jjh1nnP0p1>c_utny z!_RY=(|%Md20dQ+P@ls zw`=<}ZFnlE`66S4N#=p88alg^v8sutp$0JYSc%bVm3jKBN++kh7M4Y>06 zZ>_=G%GYX?H))bmT$+{SuV3Dzoiff{yS(wMTtzK;qQ;*DcS`C)vaup{DW~DB*W-^LX7hGEc`l@xZ_Lb9Qk))*W*$jv^3cOPjB_$1t*rS~YG#l!OYzcL!`s?{r&<4> z(LPPufwy-?46e=hlzY!ZyS(L5r519!yyeuD;cd@1`dI(p?mkW0gSRK{*~wdke-0ON zg3=8Wl9Ou-d zQ|B=~=BH*Y#;NJi=;GhgkZz?NNjVwU^TtmfGf34LFZs=Tx}^Ot53+gd1m1?NZiF}X z!2CsO_COudLpX1mPBfm7U1-+odDCR0$J(9YZSYHnonZ6U8N9V_J|z8qV}40nbu0e* zRO6BI=MgWTLalWbYp2{-Hr|*)*_Gk#1Fhfuqs?0v@b=XBMtJkRSJ|7imT_vXm9(#) zdAK(Geq%OPq4qsB=Hyyx=GSPKw~%a7#Zpeg+u2>G>}K=U6};W^KqI_K`89EAUs7C? zGnU8YP1BXe!w!A@TuZIVBY9ljm^q?0yj{1?j1SqobpvlxS`F2|r_zoh_FVBTmhy-6 zZ#h*oydCo6TaL7O>ki&}PO88gZ*@saJJtB<@y#Qx)x_!1^Cq1R zv-$2q^83c69vF9dV--3zI%Tclt?imCAF_G70leMta0T9&b4S&B-lPdg97_JY<5EBA zyhZHF@b=yPzB9?@tp|8}qwO&Ldm6DfuXmGmrK!@R(c!$YO02G!GnOutw|j>ieUHss zPw@80)H>rW?kO-^EijWu4~6nJW9Y*_uzBkR-WILj$y=5G{8VB-7JDvlzL}N&cYnR` z)fZOUy!8feBRdSQulJOv6QZ-dr^?A^{%*l{N1A?I-h8Vl{f4*y+wz;IZQlBTw~jM* z@>WCloAPA6crKK;F;~8Y|8Teb_sRQ$x6K=N^0tHUCf#*<-lPo5Pl37LCO&pJ{sXNt zZ~egA+Rh`2c~1lP;`NEv%qQD>s>xhy9zAb?og3boz520t?Dw=kcv~<#N8VyF7Ap_u znICUbw~SM>D=AKouXcIMqe`hO!`mm;P57?O+W_!3_{j>qNx#o5^+`XcA;l$+i zBWbB;mp4|4rKPTzpB~>h*Gk&ujn0S_4R0+EJ{~{7xBQ+C1aE!1k1Xar7iy$Nz+fw{M7VNr}IW9 zHM?RJYWk_^q4xFAuhDwmLUvc`#PD|b{~poA=IuuC)~wg49s51yO~unGp8_L0s&;wH zX{Uy_y}DiXZ=1Iv;O(h}9=!4F_*tOlUZglZ)bvx+6HDuPV<*&_PNa!RYw3^W(euXY z+$puxRVi+Gdve&D${Ww!H7?^!JqgSos^i|Cc5&`K7p2ONuiO zd(imHr$8q)Gv}>Ym%E5nSI=8W_0ld3Z-+km!!9;&!@=9lODgcj2~Y?2(v`iUW>#v$+a))5yTj&f6nJ}cc}Cup zJv4AAP1rY2EPpoMVpT6PkDj-{t_*JgeS6dM#;#a}R8!MaT5EXQXXTIA*u0GeZ%gi|z#DV2hY%erXRJIfZ`>WL zP}5T{+T|^$&JAxL-1FxDXY)1=yp4XN0&h9>Q}KJbYMo{}Z+TCk$o%2FNoQom4R5;~ z`?0fa-o}Hsj>E_4-&0P>x=R~aqQ~l|qKAHJP5uynI&V_vd3VJfQL`UvO+VRqV`il` zyv@C}?>d{e3E=I`l^(pYE~g>I>7n+W7}rv3>e54<&KsSqNUia3r_`FY^l&YEpr)Vk zbl$j1(o%K$OBD@on|`$QKAX3jz}u#`Jb2?SLbTOq#d}ZV@$G>*(|OZmW;W^X7isd) z!<<+vMa;e#&L&iw|Sci-ZtOg2ydEN zGkUfDUWZhbl#*Y z^%yUPx6=F#Z_k~++cx_>oetinJy?M^-?OXbQ;_yjJ_YHkdG9EXx>8P;H_6FeYdrLq zwBhZiPi~xH^ELy#^=vb-zTQ)7k6w2pXPE7N^L@)gx{bBgd$$KYNh# z=k+te+uM^f@|K|=FE*5W%$zeDZ+Ul>M}9qTnlnxD7~X#Oxxde{@3&dt?XgF8^0xEm zzr~q=;cd(9Zw;||n+@KUwVRawo=PtWq<*KQj;mtn(wq%%Cw$@C zhuFN$18?m*O-_GLWA#&Od1{BZyeC7_++F+9p3<%1t?7iHy>0V0AH2Odb0=?&e779& zL@M4(DJPiO&z{qH3#ra3r96hWkKTN?!`lMzcHa{_d256>=|rq)XJ$jnA1fZ0x0GEQ z-j4m#)bHE(+b!U2cGoHC@2T|tPfFc}$#bpbrzh5gr97z6V$}h#4Q{$nZ+U1R3l`=D~@zX=iJk<1P zv}Uc#8#7B8G;w-L+VHkZzcmhTw}Q99Pgmg0%F}RP?MQK#H_h&1-8rn4M^5Hb~((PQ%;3PT6preZMUNZ!LOEO@B|>+m57N-lPeK>^fFX=AqW~md+b9NSUQL*Os*5 zt!w|s->`XG4Bj4J5P>&kKbh~zJo3A|S<`FiHjnC3tvup--U6!|-kJ{m+OKWimVmcq z&qd%Zu+N5*Ei#Xuw<5dBqgJ83-MZ~i=l^S43f?C6zFGgC*83-&r@522=o2vaTkE4v zYGdDT%fMUDMIOBIyy`_upONU(F6@?;b|jrhq4(PduYJq;_33i(_SOp(c=O$nw9{C5 zr2HZAY`jT3lHws9a&1U{J#SJKW!&6vKm62V&j0su8+d!L-!%Pu$_Z$+^e4-jwe(2) zrN>vhyh&B0xNk*`KUOW~(etM1FvVkdyY}SoPD|d?^}FqQ>b^}*J7rS$6Pdx6>2e zZhWO6-lV(jm7n3-DYke(2&$*=LS2m1BAF+&Dzcsul-x3)UGossaiZQ%6$ zp3@9H7k2NZ^TzYaV;6bnXU{Hgn(7)4tCakPx4|R#Ki1*x%!Ic`Z`;XR#=joIY1guL zdCRF=!`mrKF8_+d+gS;3%eHv%CNVF?bJ|ywJ23rz({x@hyDpTsmbdP6p2OSO32)k}<&ZBr zyq%NqHu6pn-dMNZb@=^OFB8bd8+S)~8Mn0I?VRt7JJ8|n+=RD|TPyHJZmF&O72h?< zFU6UY9%_1G>Gbz|*}wCCi#`Fv+g*D+^$mx&^Ap}S-JOv)eu_i5s#c!zv}08dnLwEM31k|Bd62@Yq`Aft5Sw~j2Ft=sBhf!h{M~(32)oBSDCkJouJEGwL0;w z>-)JVl((*Xe{`P1+a(EakB^(}zNfxB@|{?SpS?+%`T4yHwVpRtVP=hvwKgP=%Nw&% zYj(sLT7L7M{&sxl103F(CcG_PQ-L=tPv-C0*c&zT`^LGJTFOHYb@>#yikePIYfg$e zshLxXYw|OXq+Q-5r_>4kDIUYyHIG0335T~!6W*q`oa5ea%umf))bvo(qtVh@#w9J~ zm;CzsP0A1}9*Zk!0_nVQm9!&zsOc}o4R5^;Uv|I4+hqxF<0m%48~-CRYSz+dPF3nz zlZQQMG;`8lJ_Sao*&#KvNt$`6>E|@)VJ&K|m9*qf=S|9%A#Qm4#OF@?n#0@W32zf0 zYJ|58bu+)eMb_H!?l(@4JQvE_|4rR?g2USt32!~x&egxC+y!?>Eq%)9k+g6AkXqrq zu{yQ1yK2SLdDF~X>fBmu?zbzy)BFbxZ&xO~y)&f(Z&sd$`$~#S{S+DZ?Jb1MzTV;Os)V-<>niYOgF}%I_pTjSAc)Kp)t$*kF`u9|tHg^}|;aVxD6z5uv-m!V(1PbMC?8*0% zw@2Uo_w~*2^BlI#iU1t9S1-C4;9`9|q^WB17Yf|9k1RVfx!}sYH3x4`Y^=Z=zg>`a z$~gVh8js}nUCTU@W=@wkuA-A#o{wFGag=jF(orf1Fc=pmoXPY*Rc)b9PpuQG$?$CS`7X(>Pbl813> zuH|GHPv?zJYGz|a#_2Cm=2~icB%OZ0 zNvou|Z*`4d%D%qKDZSr!IL6Z$17#>P`E8YXjcKF7)7y z`HR$?iKec`AI_VmlUN?jj-*=Yyh*Dxj9F_9Z@<4_*DW@0ZNXcY=L_O3L;q3kNoytz zIg95_VkYm&r1Qq=QiswEt+?Us(5}x}Kfib1Cf9?v*ZVBgzo%CIhO?{3csg%^laD@` zkbcs6i+Fa1w~5al{44u@YX{yQSzLiP?vDIxdXxN&Q!{5Cb(xB~@b<|&&gyOR)&ab&9B^y;d&W0J%&0A;iwteuT^!LtM$h=ACTWj&!;Vsr=ByNl0t+Wfn+XuH_?fm+* zD|oy8^^Cmn*D@L%YpR+&?1z5F%csD9Y)q%5r8AZMjI%m5J+U)-4F4%)_pL*V;D$0t6x5wW6>Ap5^ zH-NVuZ#BXjd)Se*%bV}+l&6sSQx+?KHr{-vQSuw!4*K^=t8LzTfVZt9m+0Tq3_V1d zcC5*>|8U-ztw`-VjcmO6Rw+%u@YbtSv(YwhJ;B=}_j>S_p@+=8<$Z$XQ;>EPE1t(L zq~4@j*?5z(rNj+yU-;wC4z+pf1>P2H^WZI|R%WjFeW@nD_Ovkd6QOYy5(BxQdfqz z<)gm)7n`@f;H`6uW$E`Dcc;;so{P*Q)r!lTRHYW;E^j$?WqAA9ZI>*zdFuz>HcyDm zTTXk4{w`wYw3~J=EoZ=mbAIwj_rBcw{6}AfVY*cm+RkCD}UxYhj}y;qlcP# zeA#w+V>W83BI6p(oYb1NAs#(%?8SsicB_MWEHt@xfwekm?-qKRvAMxS=YU59k5 z*@cv!YvbN;%xqORytRG)80Y7|4FhkzXJ+Khs+D=#)yh*og_P+<-%*jZIMukkF|(x& zZ~t!oY3J{g4+n4WJpR7s&6-$r#%dvdp}c+h!u8wj_jCk!d!ow<_dV5o68U?BzB-Rn zV9!!NtXn<>Mq+7A^^jWGc1r3r9cnz*F3kP5YxDWtZQe$Lw+H4_;LXa@=%=2&`L4~w zujxn6ngasiNWSoIlL_q0QT9@V4lN zJJR1%&ADs(sWp$wTdj4d=|^)q#qd_@*6=pynD0Je^EL*&jb2cJH_cAz;irjl>R2A; z)M(bF*7L?yAr-mSHz(IpOL-V~dE-~9nK^?tynXJscRK(6bS!uq_*_QbG@q;tJvY3? zA^AB&YWAb&jjKX*$c{qly1a#COId4pdwA+2&i^kn4!pJMeW&}LN@v8lq}h+gQ>34@ zT;A9%oxZxrJepeRylFCMJf$fa-VXlZO6Sk($Ah=GZ*7D(sTSkZJRRv1O~0g>haOE_ z@~|J5H?GoX->&GV*3{B?q?|5qbcSfFE5qC0e>s1%{r%en@b=mZ6?kJF_QSZOef?4% zDK6z_T++-L&Kt97Du(#EmY*6;-E`iVnb{auYQx*uM?d&6o41?5+jITya^F)aKX)p5 zH1Sw@G_@p8T;8NAj8|*dE^q8g(wc59kKygki^qJ+=4~Q)yKAWjZ&Giy9JglD@avJ{ zd8^ixvhl`lr5Vyw(uTKjr}o>`=4}#qyZIFl-dLA+FXQZi9>yh2KQ;Y~Yc$tV=f@kn zVnu3BjDBkRsWl$@(|Kbyjb>*0OL4>7e;v5yWSh6i;BD-nmHPLTwKSSj)%Z1@SUL5) zX)0=4}di>vVfY-Ym8o-YHH@oZDaE`6}-K*B_nSkz170nY`oP%=e|7@!&|8Q8f zs7DV^EgNslP%jfGl($hW_8(#MHXXc;+xkA@jf7<5jT!1i7s^}9FD>e0^ELy#bsuq$ z`<~XzB&9z2ojf)DE^l;Fb5$OiwWy_98V}bN%G>@^5B!wP+f4A*?4BCsP4jYTJe)kW z%NxJCBk4kU`~0V7KW6hb3%qT7+k-cLvUt{vYqWHlTuTq*)b#l3blzfBk@84!-(5(4 z=GXJaGu32fm6G55`t*Xkp7^TG+idW*cJ#gJ@2RE-skd5=r}I{=&hwadI&Z!eC4VVy zcx&?4(WltF%>i$7?`wp&yeBXH)S$fU)+TSUc1LkvD#?sL@jY()6U9QXXG_`4oItN!9a? z`}Rq{%NxH+9k>g_+lPL3g!A()7l5~QE$`d;o`&CbfhU#gcN#8lzTHZ9f|bYcc5|n{ zt+U_LTfp0riII7eJ}>-Ny437hTFZEdX02ivW4Ej>XV2@HVFHYW;gk=BZP9 zs5p79l{9nGBgG{T{nXibV+Lwgr`C8RhBbMZU(zmbzM1J~w~Uvx;ce{Pc}Ls4Edp;t zrZ&PG^GiF@c%-$}itBmfREz93)>K{Ia@vLAZTsmb-EH%>7`(Mv-w1Cx^;YrT%exlI z(etM1ikZm?@oT z`IPB-lXjtu8{U5QaGOhQ-j;&5HPbzK^PQfrU%9U+r&bJanjPgmrMSGMbZdCKsq?=c zv3Xkt-ezyekvGk>tL4%2CQVq1=e;A}x^a0+*`49-?xXHF%;s%5cpKDtP5OJvH=R1A zZsz-klay*nao>LQyh#-`ammAO>7k}a()3f)AD1^)p)OT4ynU!|zpvQ5-3H#e&u)Y_ zb}nghB6)n{dE{gddfvFIUNol{m$#G&7~T&5*7<`I-q!E7>#6%SIqj55-B)y(aYE@H zdTnC_+)`R-xa#;D-qievXtY#|IjQMq{&d{vq?WQVUTZoXH_nQhyO1V8Pe~ix4*b*B zc?oX2lzBTn;qBFK52Qb*R?Y@~>OG#$TfOg&{ipLrXP`E`ExYI^`0JMM@peYS+e7nm zRfr!I2tw=)yo z?tCUk-lPfCLOh+fTIeuV4=!)9GFy2JZ-4r7@2w7RXC=H%>$$d=_mrOz_9?~b$wO!3 zjonh`(P6Coh4S{Xb$e`acso1ct<#))va!=MSI0m&4mR32)ms zRp2ecoEnd&7HlSC^&8n68-m6v~_kPQ}6EZ-ZM_uMGl((~{{PzP6Z|5hx-TG1l z-ncv8>2WQ!ZysNNHr}fBk7a##M}MKbed+L}S2?_0knlEqz(eWpX$S+FwHgnn7fWa3 zEmrj+^AyV4QTI&!Ux&8~6W-b_Z-h5#GSc^r^lLoyOImZQuoktHvwRAqU0UTEXGLi( z>(Vc2jbEyj&YQGK>V$EtEA#!^;PaYZ=kRt>!rPWtGxEl|lD7J3^m{yCzi(aUukQWE zPPhvxPLD=2Pdaa!Y#L9gqT%i8AN=MlhqsFp-c}EOIQ>1<)T&2MIBz^t-wA}Q^{rJ5 zZ>6pbZ@=H`t0y?TU6SxNXT?t5&dSDKEM{WmkvMdD%c-K_?bv^ua*o4W(}cGhU(b;@ zes4f6O-AF9)=F_HXEw%Ig_)`8sRixwmQz=Tx3)9-e#znO(uB9p!yYN-Jl*Ym2$x*a@}N zRUUDdH%%w>P^WZg?zjKFqsy}nZ%M;L{Ba2aNeZd@pNioN-l3w=bE^* zyHecn*5-mw-|O&pMZ()1Bi9x4o@(||q{rorlVm5gz)VOFE^pFKx$6wt@Yd|A`%ZUw zyE5Tz>b)6x3zAXo@F#|iHrpH%n>Pj9d?()WNxr&;d)6Y2lC6D3lwpHhBcX+!x z;jQJ^^*i6wboMymdekm&IqkymwtT~usSa<~B)mPlx(;}gK3~3bW1bzGF;1pX-qwHa zd-(6`-~0QP*CxEJZ2oBa{l+_!nmeNAzhR^fS7R!>_uF*|Z>uLX!ke;I!}s$ce%?PJc{FQx-fz(- zV0hc>-plaM>vwtd-G5)-3_s6d&e|FRP69;(qQ=Ag?znw3Fk3Ba_kxSntu-+N+)1}= z+cml1%DgoPZ-ZMumj0l|>MbOXwXZ0bsmohPhnltA1+`{vh$lbZtgZ}iFAg}SgUwqD z@YZT_9q?xDt4?{#V;7p7h4MD^^#6A5x0c}TxraS?<2RT8pS|;rySu3JKe8xTSl5P% zB9>iQY`ch!wfttUJ@giO37y{^ zFR$mmUo-d2J@cKJ@4Zhp|M2>}&hwdb&YXGQnJ4W*vr~GcwNjjGc^gQ6dbpOFYyI~d zoy?%onzfp4HMKMzm$yvS&HeV3@tykH_giQ1wyFDlY2VXKdr5Mq@pSyW(b?X$%Uh-$ z8Q%V3^XG52dFuk+md#Df8~4)6>{U;J9a3`_k`DDtd2+`~6}2E6gRU+S0%Fb{Q1&d}RHf4^xulst@wYEC0`E&VQU{3@$x zX2`YFl|1JAw_7*u^Lv}O9^mc1`y24ael*(Zq3!#8O{cX{Jixhh7-dE+W-W~Qd6q7831bexBO5Lo^F-k#uXbH4|Qy{A%L#;K*xjuhu!n1?!aEpxiO z@vD;7RFu}TbLv!nDUZvWN4%Z@OHuXUv%En-r#N4!*zJGxNUnAWBLre8Q24LOb=XJoqvwZtkMdar1?K(k<&1| zo&J^0zq0SQKHzQCpa+Y+r>!u_m>&F-kI9^St@O#2;(Fd9J2$+&`yIdfjLlnL@YZQX z1KuR&rMRYNuBDdN(nBrz8K-VtU>pJ2bqV`^b0jKVYc-dDcbX?XhP9c$20l z#UuN#dxEKQy1cc*F4zOLX4jHm&zqD%Q@!Fbymg!PdHlfWDsMx;+cl#e(Z8q4x^+3> z-Af&lGiI&JTTHfQt(6Gjg#EnHDP>dc&hWNqr&B&^zo$dN+uW;5;!W9e8>^9fZ_6Xq z)$^9S6RE>gaX)XNRXo!$ydBx4qx0`&hJm+XFBIg>Q@7z+YxWrWcZoy$(etM1Hnci( zaxFDIG1}!#sv^ae)eUcxCcUc*^9?kfq;E^lRap1T(FyS!zZ zv*GQuV{e{k^ELv!ZN9b+Z<+cD;)D~+(T~epOci!Z9g{O=Eqn0uMrTaL%38zQp54CM z#pZ1!cw78ZLEfZ3rFi6Cf>_nmD#lxT*&Qd~=Z(%tZFt*yOJC>jTV4#_E*bZj`<`lk z%@XS2Ubt2hr$?iiU-~|<7;nrhX-$V^deZX7U0b^_ynXx=KgRzcu=@KUqrhA54Rv@c z^HVImeuh!wsle6&o zN}mfUuIWLuR`OI&fnTL&btxX|k^Ib0O+U59L%*LlO*VR{S*60Q;ce6V|LJJ^`?t~H z?Y;?*yYDIU#AtqYsAF=bTI=3#sWP`(9zSoXx_yhce;)PjzBX@Tz}rYBZB6##@1%n)~hP@$b09=4~u^oAzb{-dLA+DYa(T^l&Zn)00~lWbg+w%47@oqwNoDR>(;Wvl)@We=>?9(DB;wx=CQ zr=aIe%AkxJ-Y&fKTR*h#x68oW>ze|2Q`Rl~Yl<@S$KP+7Zd-w8m$yt^8Q$*y<-{M` zyp02I_ji0kf4^xa++I9--ZGs|X`f1$w@hbbc$?Iz!@F$WE(dSdOm6~jWuBl^OV3-5 z(~-K$QA^KTWLJi_ULSa^yp$&D%uq_VTPcypeNi_7JL>hy7EB=8W+d<1MCQs2&|rs5%09`+__)}p3AEpJj4R&17bdCSy^;q9b1 zkN>sJ+hp)|Rgb6q?`fvqg5KX8r^Co?krEJV>X>-3_(WC1Uo42XpZPdLDc$50EY6aa_ znzb=?HEY}N{ifL+C+z2q&PZ)|n?7;D6E<(tz}uU>pZ33}p}W)gHJ-@5H{@T>o2FZh zC-;t|9!llytCv34!{%)|c)NFDV%}n=DordVr?ghjo0Orw#N&CB&c%wG`|USp4s?FL zEy7 zm8W_N+!3{=D~+EXYEAxBo>F;xZC&FEq z!&{{*bH9D_@=K4id7A^??i}=NIqxYM<}{k6+b?gS6Dx_%rnZJ1Ow>EEc!P}}8 z8F}L;kveo@^fM1NJsR!jjhU&X%#w$3NlQ5yk6BwPZ-4&Y>95$l%>!@q9?!@dCl;e~ z+>L(TV!Ca$wWadbYu5BNHgEI6+sL8MmGhpq<|(vVS2_AAmAB<1SKMavwg9|!T)l(0 z6SnWWu2kMO?YrnOo41AF?Xf2l@W$^hq)({i;af=$HFIj>l83eYj4=Z>JE4|5nm9ew z%+Jn4JuYv|K+TGhrk}dfq51W~5feAwX!EuRyxlwE`PP3=Ic;eUp>aQNq1ou?DQL8` zRuiX3(k^e3QxlhVR9S0yJK$%V@3eVa4Bl>7TadR<215Osy~lW}r_gF$<>)6ZZ=RVO z-mctjw-4LAEdg({pH09Ur{v+k?e~5)9?!no^3cPaj8oH7?0(};rOZ;C)v4)^(Nb@I z-q?vovqQ^ocsuZsWw+S8Ed_7mFL^=#o=Q7n5A8*#z28zzM#^KIY~4?sGGEy)-cwI+S*NWb#Lt`5IWtpB{#J;W%G+6&UGQHvZ>z!E!yD@GmV1&h`D51d z^j+RkWo8~pvy(p*cA6sJehnzi&$ zOF3QMxJsj$nSREpLp}6Mnx{_>>$<$rNzGM^YjkC;;ce%)UF6(vSAw?{6JKgy@2NCl z%}lt~&l{bZ%&qLv^A@@bYf6T};k)-()n zL+4F${{N>}fwx{4zFh2nv)FF?{$Gk)y|wvs$EAMS7jM!oI6cWzDsTVs^ZPg0_uD$~ z_T-d;ym6;xYH6yWJ+v?0LU%`hsl0vfm7hAiT@Btg-dd11es4+5o#xi8ORcHv^2Q9* z%*ME+B|qb3>SDY}yJMWYsKm{C+UK=TtgYQ|_w2Is(R&_q>`^lZuJ5ub2}RORis8gGkd zv=DEhPbK}-(%k5AdE-~542;M4L)WrysCIb^&7kxf-X5I)$cG%>j<4}H{Eh(L$T_v9 zANnQDJoGS5O%F9al6HAx25M$tT+;N1YAL4_XMV}!@qP&I9 zB-9_zTWG~H{d}+Ec~f?7c$@gEub$-ac4CdU$L?+dZ_3Y0(60rQGil42k2h&|j8lhp z9_lZZx6gF{(OnL2C)Ie{)bq90d{0BC8tTu-TWBXSe%@nJo>F=HX|M0!=kRuNjkjg< z6Yv)LX{BG9m=u?$D#fKd)l*<6QU=CD{ai~OlZQKE9%|M~%Nwh(lS)PNo__s_vnD&d zol@g%!hH#N7n+BDP0s2mXq?PU9kZ5sa_h9bdAc>cjrrYv&pN#QvBq1k zzONU1Pg6~sd8oNBex|53eko7&6!=wYR-vXRR7>54<_Y!tc}umsN>_%r_dGfHa)-B5 zYrMU%s333Lb*i4FJW`xJXf*TqdDCP|Cg>;q6Z~ z-X3_c4sVfl>%NIOa&oGYmg=?+Z&K${T-uT3cX?w5NlV$JcqNbFZN#W|AK~zJT8+2W z1Kue1o=SD4I6p_!l3x=K&BNK|<4x*{)kAkD<<#VHdE+W()^tcebtR|a?Z`tqZgO}# zy~f+jWes?OGsX8;z!|ImD@b>;Ehu-Ay zc1Df2VUIT8jkP4r9wbj_JV#FUP!4ab*eqR)w*uW7-rn_Z&)wnhc4m#Y*9X6u?>!|m z)ZB{{rzeMw=Ph^Vsqz=&tw43d+v1P?{AP!@vueEEy)rRx1^R6GMzD|#oySh_9jik%HPIeE5xOqrFi?@Y3Q`| zyoF}A{D!xQr@i!5hqrTTyj`-U4sTYTrf?;3lB&1rDWuBQ%6ZcA<~bL`+lkxO{FB4m zUuwK{e!2l~p8QSWkS80{TWj(bvm;K1IxTO^Y-z*WueR>D%Hi#=HQpW_*};8JtvpTV zDyARl1Y3tUsl%9Eb8V@-{or2?#Xrw_>)*?qTjTA9tD4MPj%jOh>Uq;-qlcR3-Yi`z zZ&MfCtsM9Bmaucf+wXqxfrr4`mcRY``U~;%9Nu`b zAaJRUAyuBx!Rdhu%`EwwyHjE(br=~pz>PWKU+@dmDsLUY+x?ez(m$x7{WNRBnm)_t zP198l4@oPwYn~&u)ve)e_L4o{S-aq>ymbU`>#i@zo2PEuH)K{nrNuS9#ZNxR?$|kX zOf5Zcnu?OA5;wg4V9&wU|G$3EV>*Gi39mHZjo%bXyViI#YnjLMgHeLGWlr``Jq78j zbW+pL$umwrwZ=ogMsrHktmX2?uWG7jJe8b=w=XU4_xJYw))~Bw8QAAe+s8Y#kczg8ly{!K~;GV~H0dHMyYQS5j-h%F^96c1{Ek`H0bC$~6K?@%F zpnbn}1#eHh-hj8<_gn7#(rIoVylJLaDsNx-$hSXY^VSW#-889-|2@s|)=La9uBoe; zwv@9R-lSbfadt~B`K375rsd63(Y&X>oPM4du@2~wOMYN=K}-mHp-w@=?ZYKzTVFYxyC z<_5fRN86M3^JY!5v|mlyoBN(hpDf0y$)OYv^=SO8 zMXg!u@}`_>a`xCcdtjV9(rD)K^CndZjaRxdyzR32t54dz^#N~J&8)+lW=FX_oS0Ng z>wy#4ux$%os#4F+$$?r$=0p8nhJv>YD&{g$J8 znK?`4ZQ+)_m)g8t1m2$O+tYndHPg`C!;(jev#!QNKec9UJa4SR>KZMrWt>`bH`31@ z{Je3Mq?yg5&HZ-mFK&Fy=4}Xgdwg+0-aK`a+)L7grFdvR@w|mr4E3koDXGiUiQ#SD z*(ZF==4~i=yZ)gDylHmL&jt4)#WfyjE#p#e^sp9nJa3vR^iZ>t7_G@k(yCfBH%(@% zbHm$*-?i}po3~-$ZTY}n#okk^R@?S+2sMbh-gXeqy*H)dcbQascn`8D~uR?;qS zk~1=Hcw2GB&zwI$9S+`REN{RYcTFwzCVBX;hcsG~pLry|6!-JST}WEWOuyt|=hT|D z%t=4-i8h7o%TJ=F|iyuUEatQb*k#Aa)#z9mA9p5FFVY>-!2AkuU%1xH{Q}39djbg z&peD%(?d-UwdARug5=~XNlR;`c$sUwjQHM7x zPus5r80V)<()5RF<_uk{@w@k%W|ia#?KX#B%Bkl~+MP0Pcw2qcxDVL8T>{=V4e!(X z@2NCN_Ak+lM|e>WemSW5L^` zQGHwIJ>?#$t$o#fA2lB4ba~TcNOk|P6KeLrI5j;QT`F%wjyrUl&D*8mZN$}ecnh6s zs9!TBdZhePoNFarJ%yMI%$zC@bEeXI-a;!{yEgaR&(_}J{Q1IV;O&*?>+lv@H@Dx~ zSK<4OeeU3`IswikQXAfe9{HY^?fY#Uc)Mpzzt(wABljD`Ve)AQRoCn^)u+sVzon{D z*^%LG+po8sZu53Icw2LA3-Ok4?+s7C^?B1Usqy%E(_~P33~wJe^53qvc^ePjCchNG zTPhx;JW^cRZ>XPnit!dxk=4u8@w_Q_ZFuYX@2~&V=4}Fa>piajj`vj0o^tYSw=4bB zsd&@mjOR_$ZKV^#+vV^3`fQuGiQw(Y8xruQ*|oCgwv*EMV?27sq$;U)ohrZ7PdU6* zc42t?>Wud|f1Wi7ylr?j0dJfbcTG)Cs7{rKbz7e|cB|2x61ATd#)JNG@y zzTYN;x0w?LwB~!7a1L#ECUeH~=9zq3#a8h6c}v)#;q6D84siZH`4sRrNJ!}9#3_{+k1K+ zalFmjH1KxsowvW}l&PX)W_eT3YMpEi{`{ zB{Z(7%be6)%ebC5DFfrwtWwd2x0|j%Xui$cEbun;_JX`wwHkguCiTgvKLSNlFi#(@V4o$g1qIJd}z*4Kj-7}#;;166{+c$;`EfM%i*oEBg5NH-~8NF zHgEI5+rl0frF~DONw(%GNZ*Y}yU@hBQ%N%qd!VLA(#$V;O6BdR*Z)UezMr+Z2@@ea&Jc7a!fB(&gv;>J`wa#r|Kj}9`@<- z#;+!t zD~PL9YeOf`x_;j1Z11`lZ=5Z)XBvjLrw-lshxYexOTgQp{zF^;J@xFdFz4~SDNmuT z!%%;!T^Hl6Ky}00U8i^Wtj*g}@b>bO0Nyw;Nf+p|?L^948_!#4x13n$T8*Fisr|gs zsj1@e7~Z~d-G-qyZ_B{jZ4U?VCe=#Vb7|~tf4oVj zXqfw+=HMqs&H^)O_Q zm_JSCT58R->8Iu#7}sc*H>ne5lRS)T{E|nCGk>V|^TvwQQWeQli5uQ-`b>vKHgBuJ z+nOf>c$0dN;_Qu@e#Rw!=uf3x-dH_V74D8YRenv+e%?}bQpsa@>-E(y{MF`d4S1U| zVnn|8R5Oi)`)&KRrzWS%o3yHxcOmu6xSzL#yEDA?_}0U}wRyV|ybWHPfVa?|L;ajy zDy`>DQ;|EO)~wC#@$;6jy5a5Sf4ysWo42*#ZQC;qc+yuG~3@8;ONT?O8r9W~N@PnnaQOPZf(u9f1qNoyJoF;+?dUy4Q_9An>M|M+fG&9j;rz3{knp@Np(Y~ z9qO0rhQ`zK)~sD+ z_w4ijuQ|M(RO79~#7p$=DQmTNU5vN(xNARe33p_8`|mHFg1>J0)~^>%uJQKB%}wUb z`i!*QR|)6PmXozy-uTs+>Z$U?^uV=gdGqYf@HXn8j`(p7Z}E0Yjkoo0Hkmh0M)QX= z^oMHB%H=IITYK@R<;}C}zuj+L-nQ9!PyblsZN`+*{`Zt`H_wZ4sR#NcPiUNVsYCNr zPeIDSIJHzo@^Aw5P&20{&YWB;X+3YDnXQV3w>MATc!I;*sWsj%xwQdrR?fD!smyrf z9P0MG<95+7(U&^D2w+?SPcFmmeyfK?J+lt5Vw)Y9+ zf8_9XdX2Yrw+HYh)nZ)w4ul@+P=Cx?*3$FFRZ?}yBl#ImrKKL?d1HsHLS69~-X6Sr z*{cq3XViFG(sgWe-&4(Jhx<*X^}MC3*sMH$-Z+2i0^J(kj=OBNd%vAo<8AcpCi7Nc zUrD}G+QD1$2^ijXyKm(k&i!^)jkm6MC+02kGm?ZQ4a3Q&9n?8rB1vm@q4D_dH%$h~ zV|8wLTQuw3+Z^8hT;uK89+$?yrxIi54?DB%6sXy!W-nYz&9zdT9_nJeu?jUam#Nu< zr1SA+?bPsg*th<%yTjYrHQt_@mw-2`ZX08*952RO?j1>=+?cwh^7hpayknum+c`Df zZn-xjZ=ur*_45=$*S02a+)=2OCJ@h?l-cUg+;303=Y7*0-u_bKZ9$*Q;@?xNZr%G$ z*>l~ssq(W2KW}tus!N_`GEyh*Dx z-Eu88*QV0kDRZ)}%NxH+%?#A^P+K0u+s6GiPjGlUug2Sx{m1#=Q)x%6)oNPPv!A#2 zxFc!TE^nE3XLuX%r4P+^cssww+f7Rw@Rq7KPJ=pCe&*5SZynw=-Ew!Kx>Vjiux8*A zhqnuAyv=_k0dJub)A%(WY4Ti4O@BOZQf5|Z1zn6c&F(CZx!<1H zYk(WNsne7ia8=$qfVXZd>+r_>x&M+kYn7?vFE~xNBwXXq;c>d z;B@M~_&KIsjJE=v7~XoE`=`5X-a3P~`KtqXYvrki&L=Hzq18kEF_VwyOSqIZvwpv&;oC<)b?8ZH)@nSgTRjD7l@wi{D!w({%*ieZQgo-x5bxC^uMRf*~*%qXicBhQwUw9 z$=qrlmp7?HX&O@8%42xDcIDWsY~Ff;w@a=mi8p1>4e!iUd8((tiAkBIIM=qac6m#+ zBP);L?VID?`@GFtFYxx}3ki6$>b5b%JDED>beM-aG^fiOGcYqXJ=FA2OCBlCwV_(d z>GCEyrHV?w;qBOwQ}6?8s(+r<8@#r zxlvnd4R41&wD~Cee(M9?9=)ysZ&uE>yDKyM?%6|9<&(-hW`h&O8H`d{ed8B_vtJx{@NPc># zC4Xp~{WHIxH&$T=#+gmh^h=ugsU<&aNgl@iyh#B{hS@Tb1f&%WOVfVXk4 z*WryhL$&5}5t=iHU(Z{P>SgBi^A^*I_4mpQZyWCV!Ut{M27v~HA z-q-`R<{r~{m@~I#KQ3>Y>dc_=)01j#XdaihR9!JoMH}8e{r9iD$G+bNgSQ7Soa(>d ztemC&`Y}~q>2!+mma3~V^SHc8Gl}fV@HYG>egDbk?IQ5Derf=3nfeT3*t)|DudSX! z%#Jjj&=Yx@kvk=IQYY+|TJl?Q!`mI>dLM7|HUzva+8n@}l|N_#F>7O}km~yHH>si& zXBCZyerjp06z5t=lQjRXr3|c2ZTSsvBR_n~Hk-Gh;B8pvX=&e6Pd|0}4E0-kES$60 z z^A|R6!@=8qchuocdb>4odiY7C=EUe{T%)C&nzer3*p;NED)dVpP3IbqrdB*}nkto@ zn)~gK2ma){Hg6-q+lFq_i@m3uhO(c!cbaq}Qe3L5>49tgyh+uiI5SHgO^4E2DXys- z&zn@;iW}Yzf9jCW+PsYfZwuxW&zu4pZse`HS(!uBbh`HoSfE&G&!a=4}*sd#&e; zV(+P^ZW7+4{GqQB{nUQmr0SYDt59p!(!;gX^k{T3-ZYslkKyh6^Y$HX^L7b%du)CK z-jr+yF&2|EbgksCo`UpMYm%kyE~c)uwp8BcAF=*Lo43*6?b`bq@Wx$ePA8Rzb#rJx zZ<<}?=qg9fQh9s!ucPrlu&@4k{TT2zqwh@pdz$JLG8}s6+FF`Zcxk@usPsV@Fa>KW|(W*}36u z%=ACbuz9-_ygmD1M&7uWR9gDZnQ>~VuFD%&QA-(GIZkr)yhZNF@V4g{-!<9h?K1Fo z_kdaDzo(J=YIv7w^1HlgR<%b@sl07_`I!rB-o}Bq)yo2SOEqnN`cmZ#&C|Yk3!NDK zrSf+4Zoiyq^L9CSoA_u+ym11dY-s$U`P&z7nh8jrQhB>z#mLXtyp0EMeJ`4=e@~?y zwa0ilyoG)Wq)Eo)DV4Wf&i~y4o3{zz?YWh8c#GMcG_h8Tm&03`yOvHZo;NA8wLA0c zg`ZEq|7DxEiQw(lt#x>_^0a+_uB_YUm-6I@vo~s~Rx#c*nWc)&ikHgU6}!Ic5SzD2 z;BC>cIsW&wz5LDq`=pjmxfpL!HpW?iX*Y-;+YI6b*Fd!VkK0wdJT>MFD@ zdzQ4zo8;6~m)2I|hPU&N|KdN|yiEgdk6u-WH;MnyxTgPRd5Ya{(gd2d3)Yo-b9v*c zgxc^n>As)uWAip0yj}ZT9p2c3v0Rjo0d1JtH={fGCe(S(k_&7!`q6- zkN&8Azs&@1!`HVcZ=olp@zYZdZ#*ehXFRtqmA5TR@52AUzWVnqXMwlZUTnZy=#*0V zL-X*|q?d&rJ#Wm$t{7)##;NIv(d}@g`e~cDdEjm6_yxt@)0nePxW~db zN%a(R>>@|bn2D9jThDcW+Qa5;K6rcW#)7ia)^ZmbosTzb7v_FD z@tjxAvUyto-nP7+fH$jd!#hXvTfg3Em`5{h$rI0;lsRU1(pssOW@7nxv$`_8%|7V+ z|6udB5WKCOw6NHFYSqfhMocZ~1oQD0)2(K$v?FP4sl0t@&LRJ5^R@`QO}(W7Z=t)B zcAY9uOn%-5)l*Ekb+P&X+3X*1q_M%WI|-WGrTq%YgNEdg(DOs&IPWZj^*RZ~mj@ywyE zALem+lQM9XQS#x*)EZ=OyJZ@=yM+rc((%fQ>sofo(Md+OO^+ufy^JU!*`#)>(1 z!Tgd=%bQfC5;wek@e4z~YxA}oye*sAfH&!$@zgEMKupfiKC7n?x+BZ>92f=kBM!-R5m2cRxZv&n=^4&IXtHIl% zUQ6Bgl-AZ3lhPP9mojJ(nZ4G!E zv7iobF?B=NrrK|4UH$#mtPVqWB>7$5BxhvY@b=vg{@v>~Z&!l1H}6Z#TYEVjKW}C3 zg0-kaPhVPFDsNLh_rYE^Z)?HZy?vLt?`h~xTg@*`o_+dxqf^qFD)dk@rxd3rMwiOl zh%M9R*}Po^-c~Q(4tdj@PBGp>zhl&NLQksv%u_0FmwoGhue5nv2i~SX)POg3&dE?q zpHR(O=8?3NN2)93VZ3?@%q(fHqNbm5YI>;Y$*tpgW9QUX=jQvjpRL}thTBfne}8&4 zcpE)%x%-|nCpGgkE@}Fy>1SNh^i$K%xJLVVW9Blo^a;}JNb=};V+Lv0lBW_ky#3(E z_bsjgch4?6AN}!Tjy-DT!1etW98vkbtBaQx#7%04aki$>(y*mCb4qokc=ZtYRcdCY zrbnZtwTx@DrWQTS>Bo&uDFZ!{uB`nVZXNbNu!h@CRo;%P@iy$SI=o5e%YG!ye&~_n z^yJp;!OvTnovo~y^YbRHVs(0`>8WVL+c(zt-0JXle2upYhpcd)Q!7v1-@?~; zJiQgpsi~{+6yr^^BYLPcUFGojc}rN)@OI7c{y%bf`$LVl`>zP#tre!1YCgqyOI5wh zJmv6KVZ-ot*|ahJ9o|l;@wV=X0N!GzB(W;RV`^#EHkY?DcU_FP0v#IOmV9>O$qsKP z)_9vbe5LiwS6Q_qdG>_zGPCaj2Ma?RbreC9Z%Jeh86qnYzyh#~Yg_?ek zHoU!WnCK+WA*c?@qKx^Luahqse!yiI*3fHy0D8yoF0UW~W)yHiq^X%~jKf8F%% z8ywzFsqr>w)D`~sG*fSFtV*wdRF{6qqls7N&urAp5E|#&R?sBP-F0Y((*0JMlHu)$ z1@E}m;q8w#-kx9AWZpveo629Ee5!1z@|=b5SW@M5d5g)+Pakz9r{Qg{CD$M8@OEmA zx6RLQ=e$`nX}ecl-Xv$t>9oSyw7hwCWO$o=^N&|Ly#1-h+ww81%70H=;bdLjILTJ% zFjYUL@^<2y6PG%?omS&*?6pncjgxGa&c|D;v05r`omQQAgu~nEHQu_tlz=zsdq}dB z>J(Uunzb~#dJ4=yt;ryHm?u<+=9K)5yS%Y0IyG^gBmEU^-qQ;%J?}dXZ)enad*iaz z<-e!gBQ^KJc#LLFYXAKfT9KJU*QWAIJ-EEZRFtYzavI)F|Itm~ba*?n#@m(~wsYR3 z*_OkbbUIR8>NeERJZX7jW=k91`n_x0qYiIp)p%R=YC+y&CSdj1_G`v6<7s(I)p^WR zrL}(Eq*VoWWO$o)|I*D4Z-1`wHh#hy{d;P?*L69er$Enjb21O}Q`4i-E^jf}xRzSe zNvMbUsii!A-ZUAc%vRj+_TF!P>mi4?vunH!zd3+6D}O^aTG{XN*2>++^ycR+@%{F~ zlJ6es@ODm(w+r8Fz+1w8O5;=0^A6sUPr%%7Yd?Ij%iCXSygfMiO8t8p^C{z64G;A5 z&JSJ7Jk;^iW>?JIYP!_>?ch%?-_N<<{#xVh+D!qxr8?`F6G_!W^%PQNOO+>PGSXT< zZ_=s)adW@D`1AQ!JG`A+<84OAwdLP$1@@BsEsnq6nzakEO)d4~@}^nEZ1hvpQ_+UE z9v|s*i^JP_HQokHPr#d}hagTuUnth)T58UO{&IL@MRp?TP`{LiahEq~K3r8m8{X#Z zJosUUxASYfy>wduZ#kwBy026{6yr^*F2zH4m&5P!mZM4~r{Qh&Pab~K;q8JNZ+CXN zYR7w;mOagBmdR(SyzR5=Kdu9B=I1$l{Fn>z^Bk_2wF9^Y|G!-gK&|X46}S-tMtr3P zT$Q&D;BDevP2jD}kGDiZu}e+L*2?j6c&nVLc~I{@>&l~S-a3M}zCG6YAJkT0Bgbj( z;4S$C3~%%9I}CrJyn4TN0&g$Ot;5^)${4w=PW%7PPI=354x!pJF~i&M7Tjk2|0q9x zOlRd-urpQKf_$VO7bvH&HU8#P}37Xd1hl3jpka7W=_q-m@_SJ z%*?JT+VIxxw4dM?sMYthD|lPHunuqB5%V)nO;3B#X?Y9Xb(z!4$D6fl!`u6hIQSg< z*9+ai+a(V);EfD3KjYN&q|(e!E#>DwQ;@zI8fPccT6(DImpG({Yb8y;pEo+C zY>Z1CR^o=YBcA>Jbep&C;H^*pYuxu#lb@be)|@A`|9<0CshJ_iu30Npem`%is#NkA z-p<&kKYoC3^*!wY-kx7lhqoNNPPnhS-$RD>?B^|2x1n97s>?in-V%0bcpLt?yS`=L zZ#}`==7$^bmaw0?oNzK_>gp-9$K8eYTq`qgW9!`V#8JGN$hkk08H-0shuI$3_cJP+nKWy{X z7rZ?+YTF;xbBTbcx$ME*}P2XH;^VT1{UHN1J-jw~c;IB`m z&QskjQvT4mrXR_p=S|9>?AGwMXmF>CY~BWdw@D+eEB2m7_Ewj@966;~6}#W0Y^@gO zBunLO#_#4i@99AB)@N-&-lPfhPUfx|m)3^HH8~}ZpEq_M+7?T%5```^=q{RFX>yRM%%{)vC?PL!wFaC%aoe%_?&QZ|qOZ@lgCwgYV5 zE&^|huWrDblwT8P&(xZ=xjofWD6^~FwWR#+VJ+3=Pv4G^EMQ`^%{GF|2@^z zQt}_fo2G7zrxY%RMFL*Yn2tFax#6V|bf1=~n0O_l^W_BR8}&-njP|y@R*p z6EM8Jc<|?MwePo!!CS}I+5vB#&s5O;Mn02IyKpD$Du>qJZ;{mvZ*QFU)TK6Wqrlsv z6K-_hQ~t`Dn*HQpfcaUMTJlJ7X|0|&DFfrw?1Y*gYI+r^1Gbgp=VVs(4sp$#Tp?T?pou~_*3Pno`RH3iidV6`58~8rJjrN#%xlDmdEh6;?S$U zWb-zLyiK{ueNUzQ+f&@V-%{<4pF~a0P`@T8JuYvVDjMGQ`qne=uz4E`-WJ^&z?;Q$ z+jq!PoV!k?HMKOJV!SakwX{2pM_Q|iYx29iNtrd-JZlYaAN|Z}*V(*X3f`u4x;g%S zMbgh)r7Kf1yq$CRn`7+vbOLyr zeRo0LQuW!aJbK=m)nTf-BrWT%4R70S8*q@#+eGj-qUSB~?`gAcfY7@1S0~R%s`G8N zJf-q>{bN(M+PqByZ*R;?z+0gGFL_+vG~G(sqv@Z*&b8Fige8v@m+~;~@)nbgYpE+04R5#nbd2+!P6uzV^uM*(d&(W<(AHj(^IScJ z+*PE>{iyt)7DP)tKrrYjgDD z^2RFE?3{6GdPsF8Zg~5`{vZCD&D%`yw(gOVc(ZzIE%yXyI^i$JAO%F9a)bvo(LtQG{xlC ztfhy$rgnMb96~iSb8V#)!`nx{KKhUL{WcrC4O>}|H`bDR2=&lU&HR$q`01&h0<-a} zlBS3el^jq5ScFN}$yl(S02fX#)+JHAJXWJ*2aj8%KR4d6N#hIU) ze#RwDKXvsK7}01}p+7XvwK1A?CGGOYikcgenYos_;xW8^fA4RcZSyu4ymcFPoBN(B z^VelhTAM0v?Xj%~#x+`hziB$Bhg#}{9!ZC;mHd9*q*YR!-C7>Q+pnJ(+SBH39(e1x zrVejbp2BCEBM(21%rC_yPxTa}RZ^VUG@5xNt;rMPNy{5|VRdDAn|j~uEjDlS!P}On z67%NiCkeNy@>fs6nplz!TP>%bw^Z}2?Aq}5#gltM+uR=BjmbaVb$7vv+?^DsCzbZ|mar4U z+srR^A8Yfr1iU?cO Y19hqiv_c-2H_c8po-%i*>BrAo!fp+3H*EOiaGSTK;O(Xt z19;Qyj{Vb5O-~N3spayPqmx$1>GGzTkEU}gkKyfV`mFmB zY4)q}NVW3uCRJp&8V~&%E#-81i^=9$Yj}J2)<5oH^R^tk4Z6M{Z_@5O{WOIusUJUY z&Dxz*SBi6YlE=@Rw2GOP+VJ+{<@0`T^R@!KJ^yk8-V*i`#9OMmQXW5V$_WSUC`V3d zcdX^-jZSJQvlOqaHN5R{J$}69PSyYa!bt|g1mV+ugh4g`IAq(?yh4#m8mA?zuyveXm~sF-1kke@3$+# z+l?3AQd~2!n6=Cwn#bjhyOy#^adr}_rJU?X$||RK=gcp~>GAVMXQVc~&FOdYyKLUpfwz90 zx47?VWZi~rXr`3AhxWx=%rsoyGR@!c_Vd1Dx7oa14c?xgQHQro{j_lynm^R9Ii0k; zg?1R~*X&60xV%YDP2AJD;cehguUk>$?VeqBK6ul7#lL<3cFBOxIQQGhHQq+{z2EEoMj3#H3e1@~7${o;RsOD{gq( z?=O4Y>F{<+jki}8)#1&`Q}?BiJfZQJscP0r{pfjn z)_A-9!8*KYro?-dTC-N;iOH#1TRjEMd1EIr6`7}^4R5DEa_nsm zZ>QFHyK2A#{`b_%S@*7!JkkUtzb2k4r~ZD^bk)ipKX0izubhkFZRf8|{hq_ypK82K zU6zqIX)mdmk@9QeJ9tYz0mIu54*1Rw9o|l>@iyqu1iZBZKV(ePPj%YT_wdX{&CJyF zNIKLnQIIdv5T9{`ZvcJGHcHdNf+f!#MMB4vb5FdZ^t` zH@_-rR-vY!acapU#krQ6erkHCUEVaSn1PyWsVg4C+Y1+8_j!l6Gitm&y|MvsQY}rK zY|t-h_D?POHE}6V^%SJ9N}1`Wmi&ybb;o74&8)a)uW&b5-}^ypz+s_W-XTE!|#ZFt+i z_uhZ!+;3;qc)NMnL;m+vSu5z*tx`@-Px6GurJU7M2+h`B{M@ylx0oHVN+plst>f?i z>pc!{f3ESiU`-407PI%zwR+yHsV4bkg?1b2XU}@xxJpye@*CcM*>f-ab@I2~)3a;5 zjeI&IZ&p9aPbwzAo;T^mk>Z+@;@X&9Fb_3*(DTL&)XXNuD;~q!!F&9BSLc2^r^Z{i ziytobp2kdx{WnYNd5h_=%qdB=^t{FFw6fOlcG9%7@#6;Hy5Ig%<8A9z0lY~&itID_ zy{GA?7;o)y0;Tdc_ss)ta_+ak)_A+=xd7fY=NRKD^ECaumD!0ju~a>{yrs%)b#Csr z5w~AB%Hi$Y8gKJQKT_;HwQ4oxjtcF8bN2HVnmN?ZPD0m)`d!{aGe~}G7lyape{s+s z9Nx~W@iuaOJLApTV`0vl%bRq9IpXoWDQ9kYyQljjhdR8SU*qkQ|DNnf=Z_LnY+B*HVPqCgiDT8vS zhPR&$``$U=&HVh{y^gsMKhI(9^#y^8Y_4q_%FQJA`+;lkyW>>#z;RWmu1w4Tcjbbi zGi$(AdFue)ro2*zH%^;>Wry+5el&h5zZ92pYW#lQq|8!WQ&C#Wc&a?iS&X*=ofzK! zZtb8y+q`uIZ{5c~=02$WO=N2BLZf5WYVyZ;s;9sX3rslZ)@3bC=aOF&r^n@uU)6M? z^cdd8y!vDO0YmkE>jd6jy0H##%KSmomU5;#<A|8q4CgNX!5(fX;u~R7~X#P z{C`?MFM6+II)k^nUN4C^ex|6UebGa$@k<`YUEWwlqnSCv2;rUxkxYcY?@ z8^4-R8{TeT|DhZ0@87zDw`I4qW8S0*bB2;9o;OyJG^<1TWXAoIuQw7f|>eT%mrJ$L8BHg7$^+dZ3i@ODC4_F~w`vDW2H zQ$^y+v)0^i2dwWu#pbOicw5`~3IBWQ$)Ef^4Xve_2|d)?H)HNjNnPlim43t9FTTEa zZ=1JX;BE5EcFdb{4s|~l+c$5V3`whM(nzPIm43t9?6W?Ne_&tz{gB?^t^b{gc~kaQ zmjP+NIpRBbOFjX^+n7&m8fV{ceZbob-JbNnr{sq^2Lom1EcQvy(V;X|DK7OO`KvqS zf29mD@k({W+lrT#zTf7pFL>KDrvY#5LE2HMhkj~Je)?l{+D~_xopaZkx+E=r*Pe=o zxBb4=;VGN9e&B84JpsINr|n(GPrkiOpcro+uB7jU3~%=iUVNd=TYvC2y4O?gd&<9q z#Hq3eYI@4lp*_%_mN#Y&oj~YX`l+S5#dwpldE$n*f8A@RFWbBg0B^6&ufv-sr^I&Q zIR7+|TGNAO+ESkCDX=0nS4maqr{>yJTGNA6%gWCcmGzn9SB%!`p}UpL~eT+aU0Eecz|u_f+~cS@{d!X|0e)&s%$+Mn2xG z$r#@5e*K^Bws{*2-sUW>!&@uxkh=#xZ@D|koj)IMR_BJdzkcS(Wj1dYfwz$lHQ)l-n1Id&JCQ`4K|;mzacP0A43ZN+bRn{nkc&fk9<0^WKJe8zoG z*#ouIgXCd6R7?j|sil7Wyh*FtW8BZ1W=EQNR`M9$hOfHmK%2K=;O&7&w-eqpbE79MZ^{!%@?8ga zq0ugHnYuN+efs$S_fDI);o$A+i=K7gQ_U`NcrwkQ?kW3uOVvqeo>YG3ae2#ec9ool zxAzVHlEd2w@HY917Uhi-%P}86Z>*^4MB`yjjgHC5wZ(X222HmWkKyf$^OigReqkhd z>-9tvc+>1f<5A9`!2K4Jxz*Md<4v;*>x>L<<0p-J&3;cW25(Ofd#-ifQ_UV@Je-gI zev7FXx|SV=uBD%O^6|!s6>WGsc;7>QY4bJ;yxp=UF>m}eHMP}S-8-jDkINgor7p88 zsTTQ>JW^cGTjZ_{Z@*sD;Ub&2OTgR8rxWug?YhYVXA)gyhTd3t}ygzsFmV5$+w|(whbhypi81U9-U6Xms&6Sit`Lr7nE_ISS zUh4gJ;+5~e-sWvAczfZwg1l+&B8^9ycFbBSC*$$F#dH|6mb;*qa+b!}+ z@V0663+240(lj`+GIcTDT5T7aiIvLRoZWuW&F1Yg@U~=q0^T&IAbF&?W|A?UV!Xvv zY}VRRdHdM#*Bsu)fw$4y67bfnNygM|E^m>)^U(Iaq9%`@x6seKr=t1w!XE$og^Bk4 zb~$+Ke%XuVyr(g2gw8}$t9lAKGHa@*@@V?^^OmZUN*=>okIq;Br_I}V@V52(Ch(@2 zlExD{E9UgyZ*+!MX*GYTyq)%|_x+pA+XV1-!z)eTtrbqh&s!^Wn5u_TdHduoE3dM7 zn+V?KkKb1Qdm4Isg!(ln)qZ&kokpp={qpp=f3#hxN=Wp%3x6RvB@V5Sz|3kd-)bx`lrOePQ z9omndx6o|dwYAppwsh{1zp#0m2Hs|LcscF;X7!N#zDkw9dJ3W0=%?l>NIKLnk220`a^lJ2bp~`nb!EvU!^U z-k#sQ9rC7`Py6DH{7~z6O6oML3U~}}@0q^H`Te(<;O+L#ucUoX3-p=%&DHea=S_2_ z8jq%v+#ZtV-*sdshPNZ%cy_pbzs&+~D`z&~jekCS{;)IK*r1jsPY*Ts#r#rST1$VZ zb|=p&)a;OPYN0UQ(8-pX06McCId5TI;rF|yv^O`N6!CWW;S@c;m$g|F+VkX z(CC=8n*16Md!VkK0wWsDiu7yZ8V_?ynt3$Z!cf7I)zq_>v@aZh2ibF{od{TeWbbIZN}XH)4Y)@>d48reb>T-CibZhRn4wHNA zZu2%DymjgQn*TkO&MS1+W%@NeyS$azVN5N}?wG$E-YOLhZ^L&zVVlj{0`T_Of(E?t zQzmKdjylw@Su5q##F^jajpxX0TtzK;7?(8jP;2}e4|Do?3*9v{TiWpUzE{sU&E{<( zczfvn2E1808@{3JUs}tF#dtKen4fj4r@#oEp<3#M`KjrrriWVUhifH|o;N9jrjyEA z!`m(QE&sI5+amC`q2KH7d#cRekUgn?#$#&5tS!bHD{6G;P9=U=OOum+mp6Wu*(6QB zM;qRDnfI)7zbyuD>z35vO{yzRL-MzBym|^!hnk&A9w{!>N)?yty1b>TXyq}yed)5k z2iW)967V+p;Q-#4U)mi#t*rgLNu9UHU6;z+uCvA+ZS%Giy!9LOhWnmM)6N~|T^(90 zbgj!9GlX&_`K7pKckG9L#wAU^pEqWR?AF|G`_KE+p*C;Jz}u_KGx8SGKPM*XnEZa; zq>7C5q!`bwL+ko^i|N*yzv1n?fBx`;HgC(p+x?Ff0-S zlRWu&vnm?imMlNkx!+cRw{=6_O#7bl?;h}zNKKE`b75{n^SHdRL+a4X8b9+>OKa)j z^r)q^E^k~#%_>qhdZ;UT3~!HoW%o7q{k9UkUA-zXZ&IJRfV`t-ZU9B9(I*OYkJW0rpe~nh2iZd zy&w3w&D$#Q)_=r>yWww9`cJ`AtL~Y$vcL6r8hQF{C*tQ#sv^ZbXKHwR=Ra+8-qY3K z?fEMk@WwZqwHT+ShdR`w@iRX)*HY6{4sWbZ%}#P?*7EbFsZREonYxn4@V0jG()srN zwg$XC{7eJhH2tLVa9^o%N_ncMP-aD`6UIaP(fIwmX)7TF)C-70`ya?z?VXYV)=ZyuCW6 zqx+r~s8#o_Yvnxt`>mBb5ACf~-o7{UV&~syT@Bvuzjg<2CzQ{gbT3KqP{!zYdCO6y zQqkORH?IBAIko%ko?Ui6`V+?-d(_N<>-#M@qH-s#d#Mg?ns#EIdJcn{bv5}V5A!fT zb@dRW3{spKG+NV%CQppV#f{mxQ)+ss>7lN83~oPLIN;$LZaYOs$<J1B z{crvCx7`Ny`XA?hJE_Lo(l-n8mSbYvS7?5XpYtik8#`n+YI-EC=~h}B8rSp2%-j(* zJ(f22+i#x2zs}m}t^4id8gJvKbZMRUwER=BCf4u{X^;M5ycO7~;qCEbzctyp-%hFV z*6Y@cyyfOc%Fj48XBdCKF&nj1ksj($Ki6usp0~(O3~$|6Z{6hZ_Qx7;TRV1bo%>BQ zy;Pojyrt?eRi09LyYMgnFwEiY)EaL$O>e*(-)?G7LyFTwO+Pg~lBS=Ueko3m%Ntiw za}_mj1Ie$6OCHvW(R$uAop9GySLS{j^^vYmJG}j=#@ki5H{i|6+3-6YPd{yIHJ;GA zp?;UQ&6_Lkio-cGOa)_Zmx-cn7#(_7nJm#S81p0vD$R;NGrl+yCX&Mj?t zd+*?dLmb}DsPXpfT?u&8OpM^!tqG2Wzek>ZuE3~%Rs>Nnqa zcssMk+wDENyYFe}jx>JuNv%0G`l(r0iqk{w^2Q9*T-6Giy-`cO`FUdoYGz}cnx2X_ zybb7b{Q`%#vueCuKd&U-tlqNTU7A`hZ<2# zZ*%V5&Uuq&c;4(%{i4=!(-%munNyq&u0`JEl!&Z+VC;=($-@r|ZtpJm2bi(2YOuN0^S~|!<&^S`Mso>YVID& zzu$82NSdn4Tc%D7Z+A^Q<3;Cw`)iH2JNx%c`<`anOLFG@yh&%*UUtVmOXcnSDOY{o z;qBZSZ>yHp;Z2%$Xq>ywp{0H_wfwwkGPD;@sl2_h?wA1%Z|Bu`8~;ch-lR!t-f?=U zrJRgw_|fE~hnlrq-XteiwX&A_&&M0jt5V&(r=QyTBlpiw&#&=z(coU~>pkTjV@}`Y zEhbwlug%AswQIxM!8fiy#JS%tsPXpNiU8iEPmmP1`f2+fjDBj#&p7?m^k}r6x0ouL zwUUSNP|XSWd6P0TgT_eVg8o3t0r4H#Y2Q<6*W62)9;t3TZ<<}Odb4!7yj}gG_48Ce zaZDHR_RQJ_ytS99hMs0TZ=n^N<#&0@bf)HhJM^fY_yhauuNS(4w@uGB;4Sx*xOeIt zb(!DK8=dS#iqj+Mn6*+)#-;o&Z<d#{Z{K-O`#tRj-sWD?H|=}M-db7v zd25BvJttfDJ7lS6KX1~i0&&CJ2flONJvMLM!P|&y0(eU`fzUjme$A{}pEu1eG#)>1 z31@3~+i>eUCfdC90B>)+5Wt&ecl7Yr*VOdHXelS-)U4(5CONr^nkPj~kEBCu(XY|0 z>*tM5NwXq%N?loNciZ;BR{oD6{+UBh{ zc)RlDl6bRvD~&6uZaKV3)ys^#yk)9xc-wRI>{D&t`hd5om-la-_cYTS>ax9qx8xHr zy#0K{iTD9Y)!%>X3*H9Z*nl_9T`GC_hDsiZF^ym2k@EYe9g|tJmL6($tC=3xQZtWx zze!Fh&TO8p3~v*D@O%7#q$+Ryz+1Q18}P)u||Ik2ue&~J-*1D#Ter=1c;ob}H@fccEzr-sXx7pnqeE*I<1MB`&00=Uvm<(_St0%mDy?wt80dIHCNX#4iX=SbFt(AA7>CNR$v#Nl{@b>A) zFZzPb+feYf=8nX?NuSmN{nvfFnTNHg>7k~ldJ2rtnL}H*LBriiJ%r{d##@2O7~XFB z`)@hC4Fhj8yA6(iPo>_ZxHL(|W7cZ&xc6JCimWba)}p3giqk_)KefvnzpBx!Ld~_6 zxZ&*&A3EOo{gC0{ZNi)cyh%M+{S^L0GmoVCsi%j%QMFOzHb|-nHc$quZ^x)@B${@u( zyEDA~YJR7`w|N^0-d^r`QL*>5KtF}=x0qVe+W7m;Gx@?h4b3U-j(PmN(HW@?Z~M&N z>&G^47lXGu=QrSu_h*deUZgzqNIG<_oIPmPy1Ypl7^h|x#;GeF z!`r8J`(}Tew^88jj{6$$##-BxuAYMCe#o)wGV8j$Njr+{%JBBe(a*la=Is*jwxaKl zV(+Qcb1TO=PyhXvdl&2^cP%|{k)0Uc#{8<=`)%GvgSX2UZ-=~TPSfR0vnujLl6+qm z+GjD|ScRGsu;PZdf7t8$r`f!X0dHL&O3WMIbZYjg(J^bSIn+I6mp4sUsXW}Brd!tK zTE-<^j5k)Hma0_ZhPMMYp4!jmZ7g_ucEHf`-_y_ud-hnDC5^v)-a>aB>X&w1j5jHp zCvJHAx4ZYc-{$R7@OJC6jJ)x4LG9_eFh9)6TGaIDd6P1b2g$>V&C)J!QnylE*}38E ziBFCHjLq9+;BD=r4R~V@)XE+T^CQ)g;+lTA*5!>~rPgGYJj_F#$}i<%+~tj5)pSKa zHP=?O;qANk{=xb4({bQ!=|#iJe@~^J8J9HoNRJe!hq`(SnpGMPGw0T!{kXhoGH5)! z;VZi~ygjhK^SkZ)?Q-xo`HCj<#?!I-FMKy@^0@aKJISFnUFE3j@|L4YC8yzSPOrU> zw|N^6-ugb#LcC>~LmRg|@nXC&L+)-<$V?W)G6q`01f` zdDB#32BqJ;r$b*nY_84QB=C0I(+PNEZ_U!7w?V%9EwqzRf9Ql=-a<1dJN(d zm(AN`@V4~g5#_w6nw~YD&{;|Te7s2&n-!1eP1(8OZScuobbh~g3V6G8T}iwtdv5#Q zKDU1dZ^b6yxf2X;zv{BzYxey%6}XMSVXwUmb*jh6B-F6D7~qmvmTwc%~! z>R+B|^EMs4J+!_KZ=wCrPtE=rr=};B)_gXrr@)HrikhBKohlD^6q?iJP1*%BOIli6 zi5uS59X{Ln{~65yZ?|r%!<#2(+xJ)Ohkhwe4>kSN^ib17O^-&qym6J3IrQ9_Cq`@P zO8H&hG}$-}YF1&qlGE^Z_{sNgwC}f>;O+X$E^eLol>JMZvywcTxSuyo26jTNSzD&Z z|F9l@qD~l6%B8{ zeAkiA&wrZ(-rg8Ls@QuPGXbm5rm*Mdt-b6{(`PZ>G?}e)G`w}c;_xf&`)w|GyZ5Go zys?(lTWFj)b8A0uxvPifmnNynla@EBlS2I zV^=w*RE)O*ofzJZyy{`+{|`7HyiJ^ZiTj@Boy}nWa2r zK=R~>yS(M7Qpss}8~DB}POy1f3f_9p3gC^O3+mARxwg5dz|Pqf+ZPU zl25?!wqe6ZoS$#G3cS6#Ab__rc_2ShUB;=oR?=zjH>pZ1#P#=EWVeR5yYAZgQv3V2 z)!^;!`|I!)dUlb0HvBbdj(%F7w^X|>mABc?ANfn0w>9AHihh^1&U>1hTjsBx0wba4 z8|r7RGIcTD3hd6@Z@=HC*MHl*T?yVUUlPDufgalKu9&qsPRHdfN0rc=nki{KtXqsX zW{|YiiQ#Sde}B~ZdC_aZTepV;c$4a;id+4+v8u@*y0$fW)9f_VlaDv66T@4lBl;|{ z@3*VK+YCb)*IZWP?VeqBK6>wCjy-DT z!1etW98r1qE*o;W`LD1sp*N)%t@`uNk6q8 zH#%8Giqk`=D{+I{yElAlOAWW3s=OUn<89QcI=nIG_Nwz;Z`LWaeJVLgjfegi?dL5f zv$fXn_HWBCf5hSK_!@5=pKJneWlr0gN!#hWyh+aX9xujQfiwEs{r1y;SmW^ahZ=7W z4!>O0()=cO4`fz%^N3HE^i~Bf1ktK2{qoB zn(Cfi7~bw0^{XxpZztDy>w9%U-aK`K&Rvrybi$gPe%^9a)O3}@qvuUi-7^`(+wS9E z+vMn6}6_% z>M2N7G;ztJsUG7At?Tl}4Ah(N!ZHt(al_kp{&UGPhqpi1cpLC$0^UOV44qhLevLmZZ=pL1^=oz{c}QB? zU0W3mZ-;L=`U!`(vunJ)IC)BIzNgk+f`0WUohIX1Cm+;p=uWu{&DvtTg;uF_Vt89S z^mzR5jNbbFkaKFh-L`2v9;rAHLhU z-~LkLZC%Hy?)_GP?W~_@YhMj(Y5I5XH))j=m%0k|OL;=$Qr&po*ojo7;xW8kFyf%! zIK2I}#@oc{b$DZbjrR1=kPYTfm4`Vct>;b3poy2cBdMRXyh&9mal_lcZ#nQ;hqrTU zy!F040dJljf_P&do)k4bG1}!#%9f*gj+|2erSdlLbGN<2;qAN{Z%=iZ=Dw#==Tcmn zUTB>8rM2|LXgzN+Riw3N4Lh^&CsE2N#Y^R_+Xv5{;_!BUjkgW63i6hlhury_%bUbT zj(DlO?X&M~`#Zc{P~&a>T?Ki|F_|)RwmxsEK5?b;_QS8Nct3dC^0$9qe<6OJ!^J(O z$3LiLf|9D1AGlP9ovK?+9_hC#N(JtS7x!IP1Fp(j2k_Q;UP-+11gWJ-YCO_fP2A6$ zCPS!4+L06w%`f>& zUzYv)qc(3{z}w{y)ZvZ0kn&5OP(SlXnv<72p>dZtX3*>+#?Q4H9XbK_@A4K}QB%EA z(eU=8J>T_;&0AOS*1!Ku{d>wjxg*A@H6HrQ)YVhS-Jz5}G#)bl=f4C&x zGpaF9Wke-ZmGP@zjN#y`+n;I-mV`s%m1EgcEky^7oC>3R^D}~ zyd5^=z!z-ZdV;rkD;n@7O;X}eic5LQjI+1q^2SbbParLCp3cpCy8FRjxy$CQ7kC@? zctPGcB~K59-$CZjp_|KFjvYxk)AA-&sl*L$9iMm(|3Ity=UKhM+nYmX``=Sf{-(1O z!+9~@G!^-)wwUQ@@<<-`>GCG6is|+(-flat!(Z(Ctq*v6cvS#zQlE@dNA{nUXFqS; z1$F3-G;1}U(7N=yyz#3EwYlGZG~nEMHgA2w+omTI^OmsxCi0eJ7vzdsQ!B===Z#gU zH62zwhPQs3pIL45)(^a0IbzO^_tbyaDW_fc)4_KwCQr;-J#R5pDr*gI2Oa-g{J`ky z_iz2d+vK&0d5f7a*HTM!(0HV^)l*;wYH3wyJV(yZ9!lkH%U3VLKVYcxHUPZ!d^Ry} zq0?xVKi{3!tQ|>p{k$doM40>Sn{%cgY`>=i!P_&V=C`gvnEt}39-{dU+x-Jh>raMk;51bAEfT0!0l^b^En?)+(a z%iV2Eem`#{R#U~}H@rRc?OBs--bRAADH9jOzo(i$Tg~J0R^|^a3Y~^@%2J&9{k+i` zsSR%rf8*&+Hg6Y$w|*NF^Tr8KNA9KW-N^jZnp!SznpKh~w7QfhM?4>I)~*e2oi{A| zvd!Bl@b=W3b$H{OPAyG?9_mznO&*Phy;VDKs6mMTUC3RBgnw?6X$|u6` z_H^HW|Fq59Xz+IPt#x>l`e`q5={8{hE^nGu%+Ly&JyWwclIHH3PL1|_muq-i{L`zh zws{)^-sW^#o#wbz}w?}mX`mX7WnK0-Hn=ARZl^3wnB%gdUJV`GKa>kDH-1Oxas_TZQdq> zw;L8Ufw$0oBELcpJT8e0Q6-so-t)fMxmK z(^M1Y9HjlS7CqGTR8N6lWtCRY?2S6KHhCvplH;Uu3eB&zqD%Q@!Fbyp8Dmr1SGb zW`ei%D--h;*>e(>H1qNErpeG=JbK9G&waM)(YoM zbEc(=hPPJBKjr*>@+shL&479O_msQP?3#UQ{LG)VPUbD^ZaJAOcFO!NZ`_@v*%dvE zvq~v$c>Dg*&7I%>HWj=rSelYI){UjD{xkhqHJP_qJ1RmRm$xjsDxIC-?VbT$oxiVU z8h9JNtw6kGF$XDU`4mVvwX~`+#$DbtJJoosX&Bx{{OPWV_V;hMg0~KX=cjv5ty-Dh zMN+L4Gim6vzW1A^bB#xu4C5|u{AxsPc-wJzf9L1Z>ELbaZH<_>kXhB2H{Z@f*vpl- z->jSM{Q0aI;O+kHyLoH6YqOUtZ=*i(%|F}k=}ho8d+37X_cY{_rTG-|XCG>=Wn81V zR?>}&F=^M*?&!~lw^Bv(p3WY%Vu#J!Ebun&_8PoNJ8CR(>GKlKo0K_ZGFjxzhquzM z4R52bz2^%yZ?nN$k5_kb-lWqH=S|8i#j{{Zs>OIRZxOpPygl^hZ)V!O%>i#ahAm8f zPucmdr1R&kUej}Vi^y2R&-pv9Z7ND{E}bKo0LHrH@t1U{%g*^U%vpnO}RHEZ<^DIj`9?KOJu=&d%`mpU?l(`TlJ& zcw4uo25*|Wnu%#Vnp%>_H@}`YsiG$C+m$A##>2XL-ndFr#qt>54!(Blk@o$z1iZ~{ zxg`JhG;m*D-=|qKX;_<%w=6pM&FSmc^Tt_es#qSw+o304{(GCZrQq#`@l|-^Phr&T z$5+S7<6Bqqr{hh^782)Vs5Lq1r{>ypyfK56nVyn1yp4R{-QTl$TL#{`uC2nG#cji# zAjPe^4gRd-A^o_#g=EWmry=>ndDB!cP0#Q)X8dQ~ZS%GqyuHw7Y5wmi`>)q2yS%Zh ztaZq)eb>6YNg1TLavFxWtsPn}uJE>guRTva@U+uU8Q**h}WYX!GG%DkOX;ce3T8oY({mgm}p zIaFmn{Ce}PnCCQDD;;m#9W^ss+VFPmjeq!n!`qn^-gf!4B#7jdLxvCXdS-S7|gWCTPRk=_6X+G%|0#)1aUC593+rWZtr< zSg)KeZy9xB?zdsxJ~GYW?LQUX9`C%|eNQ##P7ig~pJ>+7%uVVif8N*$wX_R*sOe{% znx16dq*cryd91TAynXPd(N{RUon7JW&KXsBW33Qv_0#a?hUAR3Hh@{~^@ zq@rf6#v^qo#ig85JXhX6^?a*N4sYjIc-!!l2XE4jq_{M(BE-|(Zy}x6Yf8EDwr=7l zPjGmPwO>piIDtxlRC*d?((KN3ysI>{H>o)FW9@A!`p=w-bT)^!CQ*D zb^ke3)ARo_Zx$!9&dc!Dt?i0=4sRD#c9E!G}W=h5?~p*NOCnzkmtv^L%S zCS^;A8{R%Pc3X3Yw~H&hz0h}M_vu+T~4h`ewHB7~Tf#Gk=Z4+a(p=9#|BCH`bMUmf~FNt3z_?c?+rHyCcaT(u1Bi zsk$<5c>BMTCZ6f=c4>vTSz9CX#@_fTrlv>I^ixZI#x?$A-lPmtoYko%KjYL~tI=Fb z&9yFXbZRuSC1}IjaeJHv+}{5EZ=t;_bMe8=CAD6d~Yzb=( zZ+i_r`B?y0{{5DhS9lw+tPZ^Sex@XU!rrTL&R3~?3YyIHNEMlxnttY?rbp5mzvRh| zH&!fZ!`nq)pMJ8#+Z7ewI=xthH!DxW*)h(^FfM8OeYGYhJD8r8x6w zSf!sisp-j%H&!WW!`supA2itE?aB&ot!}tIyZ2O@p4Ddq&w0j^c`I_~zEjGTw`EuS z{d9-7|5kW=dW8pX(kDn0mpsz+=%@Deb1ilG6d0k_RFOQ4Gp9yNYo$1I(nFmqZ+lL8 z?nsBXt17%b@v;YR+=WK-mY|l_YNjXUk=9Cnmp69hn~iI!r9AXV9da!-*J^Y!Z@!u7 zw>mfPX{#QO9_R3Ob%nP(ZoI>NPnnZDVqDVnOPXs#JY37YOZgd3=8c)TN{VYd(ppWN z9%|N=bUNOcjTNcsDQUyoQNKTBwZq#r72a0dQH3|=%tDimEb?n|mQO+BZ0w$7-XiYq zt@~|hTl{rgZ~yw+wH4kbzE*>`5H>>AMw~;{pGWk(NxSfkXT2jSe>&cz-APqSal_jo z@BiHZ+;1D+`t$Y8@b@|N9KPCpP??mMCgaqSCsy3ImQ~TySOHnuE92cUR%f%F}Q`NO5V`zJBK69GHi3`l;!c;x2D=N?May z<6(aKsp*k4{nYd)^F}8#P;=&tQ-p+l`Gy7I9xH4}oz}wC@s_@1h8jI#kG(D%| zO_MoKk7n0;-U2%}yzRU1$@mSIW!|m>ZyQJ5nf#sx_Rw%QN_jLB<62)o*HWkBEmn1Q zlC@6e&38u$e#6`L-c2gk+a6`!T7tL5_j&MU-JexiqKBGOmEw|zeo6bTO~;#5CE*mR zPKkLWt=XxjZZdCDb;hMCR)=rhZ?BJivI1P0w^rb7MvJ@Zd%tmCdFpW9vglSiSt+jR zLC+hjXf&&r{D!x~9=mRd&0A~m)_+`N-Yn)*f4aGsEVRoTJI|sbbF!93bFHMMJbK=w z%u<{cOCH19y$7F#ztEw4zqJ8x9UqL$o75ZQ{OmBEr*?VstzLv(XnIcOE#fW=Z}0!+ zUibsgW!~C?x2>)2-u<2?v&TO&5%F{~z4_T^GH)7Ul^(;}-@R$ z_KqnZ*=h6E5xiYDr3!CZPax!UVy!KoLe@L=&F|~iOe2}Mh!xHK_O%bZzMsuoC-C<8 zhAO;ic0rG(XN`wBv)20iE$dFQ&Y#R%#Oj8(3m;ry{rfTpp4J(>t>}1fy7!ddIZ$i* zm)Mi?Fs{*34_q6UH|~g9>LlwpdvJN<9DF-TSZjDY@4^}Q1JC7O59tElrr(;9H{U)b zKPRK{X#7$h#^dtF%tg~KZ&HULvn{PPyq&b;ph5O~+7-MFc&yI6Nqf&ao*i!?rzY)M z$|=Q}N6#BqrO<}A1^?)czYxECzjXs|Z+5va-Fup%pPIj>k!Q~Ac*}Danp(N?Hu{#X zpS16{?%?f-nN@g8c%vJbp@&*Cfg*Uyr@+ps*&*X0nmMVZob*UK{C<C*zW)pPGKg^SR%o(_vS>n!BKu^5lBI4O+i) zz0F%s@OE2|``!0cnv!o^nm`faUZWZ37 z&N(s36B3v5OYx9e`ui=5>LH!`uFaLVacjOZ#^$Xzc?gE#Ip&r`^vmYz56x@cN6 znOu3h{g<~MX!F(wyglE0jr*Qzrplf2y9maa$2ZQkzMB1TE#q-{V`f$i(Xn#I)H`ba~?{>cE*B-sV1Z0{%Vm z@~^-318*~*tHGPpgQm||9;pZ4IQuD|0wccJeEqCN?VHEf@ABrGLD{Y0?aKBS;1AfB zdFv0}2K0X*-FxcWv*cIqrQz58q&&WH_kQ!upz+shckC^kH)f-@c4v6I@wa<^&VEk^ zfVXB#s_@1)nwqtIHSau8{Tf&v*}lC-UfoV z*PgG!n`U>C$6`MEy(HCTTP5)u@|ICohPP%Luk2&Fi?lpLKrr zSw4k4E3$gln)`Bj<5wegVtBjonE%(z=IsXX)@#T^$?s`Ky*0eQS?8DLnce-y9Z^f2 zWF2>T%c!E^ZNT)q`rEt>0dKFYtifAGy*2Pwul(8ZChe$R>}AKBwIjn@(_@!8fB)N1 z@V0)32X9vXx-%o4tQ2SePL!edUJUT$(B&j@b>GEP5QZgzugGl=HIk7 z`8{QR&CfELy)ciYS(n<^&$ZO$Q(!iJRf4{t@7SgyRyJmlS{+Pn=1ZyoQc!W(a6U+p_# zDL?bj!?;FsEp;+)bW&?7YCKX-#;JXCa;?i7zpBYBd8|C9tfJAZQt}(#{xq-ew`|@MC>$-L3o*tN@>#YxTYtCfDkTg#7h9BA`48oZ6WZhi85TI6XR6k%G2m_ZEsdBr?p^sAZ1^mfPr-K; zD>fGG@|IEOhPS_6(dP`Cx3S=@<3lNV)0~vXBket8t(24TboU!GG#0JrEpXdV~9(@{X0Js_Bh+B+dNvXyO_Vb5du=8>>hi(nC$Zua@%Yd6OzKGd2CC zxZ!Q~Av5p~*q8sj%q`$;)5I#gF^{iiKa!Sm(l2>p#mlF_PYtzi6|SZB%_I3a8EX2a zxXT+m2;`WcY@2S*72oE7^rG7%<MLe{c}WZvkc4yo?DmUShq$?5W@ zS(UnNOQ>7( zZPl!mKE;}~8js5xD^g3@=%J>cCq)l6{aNT-d3)!{pWWN$Z7O)1_IN>g%VP4(>GH;} z?n*kGH{~?m`uX(J-#pajZ5ntR+-+mJ_f*+K)w@pmbo<8H1J_EL{_J>TbylpG&Xu>p z_nz`0o3~rRTj$vkc;l;)#Zk4t^2YB{sI4BVexkGXXg-f|dE*2mE$v$JOK~ZW6nA-J zW@%N5xZ!O_lff%&-e!TfTb@bDTZ(?_!k*Npo;MA>QWeHyXx-*1|1Mes=ExV%ZukX@8=8s4UDdUlJ=+g$MW*Wjpyi_O;a(%!``GkQqS4(W>qxz z+v8i`<@|c@0`S&vTY-3!CYeRt<&9M|=SzQ)bzI&OCS!Q};yZ?Z*}mTvg10vZJ*j_B zrCL&)r$7%iJ$dS6-tz1u>uIFp&9|$P-|%)ozj3Q<-WGwk4Yzslmh~z4?lGjFWZu{b zbx1|c+AM0NDdH`#)&&wgkM58M?`RPss^&LamyUWFC$7ttI7@@{~`(ca@Y)!y9Wc598GI zNLunU9+x*}rY?18c>8+u2|u%WTMFK8y1fc-((ZPpxO=~m8S2KqYdvp)yEeR?aK`ke zHgC(oTf0~Phj`;WUEUfCSKNiB=Vab|vn5nCydAP=pWAHSmV>uvhdt%Kr&3Pq{;c_i zN_nKXlwXQVc{K6zDM*>6xF&PaJjuMp>bAtK;q8UzzIaoGxAlANdFnx@oqo#r?kl=X zJE_FW19y7h7V=Thtkrm8jVJti%d^{%UFU;asY8R?q`%$%%L;CLlzBU&!rQE!9=u7r zjx~8r9(t&y-lTXqZ&K!vc*u^VwfXQ?s&05&a_r7OIJ})%;cei^r<0#kX%`_=mDc82 ztC+k=lL?9E!&|9C!`mmvtonq*+gTOfn%`T4w~(C{Z7q4uhc{M~rlIjjYc=s)d0VmZ zqNNURO)I=T-fVO7{ifMbERXNxef{jSdoINRryjeRn-@l#romFid-p;A;)}!S!^?gs-Gc_mXt2KGT zdDB$!?cCST9;lfoJKk8Oqz!K$ocxP-IlP@);qAp+>cE>c8LR*3znskcE^nHy=%Hq2 zYK@2fEOf5Cz1sY>(;VK;tMGRJLsfX=O|H>dOo??h`6a&;cX`uP$-=`9seNmaZRT-# z<5#IQ6(xTuZtk~!>%W8lUH|s4_nu$jZDpHhliyS3r)F=yacQlTC&VwUb$MfTYG#O~ znV&jVe&%s`<5weU!`pxd_Pfiu-!7=|Hg8f5-q=qTlaX@r)5;z+nmMVtwtNb{Rp@7C z##x0PYWk%(Jt`OWm9(BWDYF!3#gfPHcHgLd++TmYxWZfeX;pY*o}y`T8`7Jew~#7DSewjS zta+ApYIxi9nUAk=?zc-Sylr_jC*ET1OUmOL&yF{#lX|h2%$wAevp;v|C=o3*Gw|RqrcyLJEWgFq}z}@E^i^(5_VzU)78^I>E3UbS9t6CL=D~&@)zp9 z^6gX4n{Vbk{Tgyy-ZJXY@b<@}zB0zS->#_e_HwtaaoSOGx~({ti?I+zTcux!0^^$(DoZ0-ma?fHoVvK`MjqgT#*yzk>d17I(gcf%o>kW zJtWS$)KX6Oev_Pmal_lbt$%fe!`syr-rCM@1iS^#q3WGnJ_YH#q<9vcXliLZQg6w; zg=DrW8s1La{|}#Yc)O;;+vaEfr+Kp`R+Y>mdo;8p^X9v|go=i@9{W$Z$KmbT3U90X zzEJ%4)ORKtfB6(NPG3*LiPVMXxV$Ay!|>Ma(Bn@9ZyVnF^YzW}_c=^kv>Ujl*}<1^ zJW=4&z?~~_cYOHIhgE!a{dOIAdts>uZ~P=ub7G83z0t$9lBS{iNLw0Jt+QtZ_5_O7v$ ztbhOXpwn7|xA7}Hc+>O{%aiq~>3P%aDC=%D`K3P7@usP0c?@q?-*Lrd_Iug}ymf!s zgEuRG&3jDpOOuiOS;Y0cv3e0YXFnR9D{qSrX@=iWUVcy8g1484z7+nRTHiNT2K*cIc{tVn+;Zg~6RN#h@}dFu$?Zn>+Vym4Py=w#kFN$M;* z$vS5;Z>%1&)6!bQ+v>JG@fWI=e_rSW-g>;@!5cZJX3vaM)5Cp99x2W|^ib0y>GCOX z6}7ZViu_N!aJdVtMa7x1?E-Wt5cnoL5UsaXoiSzq2lI`>_x=PmH8 z4R5_Zv}T*lTUYS5y7`Xq`z_WVe)gTD?|kS_=FPVgUq3s^daa%}c9Nn)!`sgv`@Zx4 zv$}z|@na+K#y+X}-G!v-mo(Q(9>z8P`trtZLo|0PX_q(28PZ8LZr{XZFE-*4T) z+t3Fh@W%cFd(QN2WuLnSZ<=YlylGbXdP=i3y#0Ckd8gaF^#E@jTfGwgp8EDiKN+B= zhngO0dNewmw~*>wOD%Oy54GfHJXhXU9(}@lZQgoQSunKw=*PaV!%p51D`o02^F z@K&0Jx!=Zob>9UxZ@s|V#etUXy*Jteetq*t`IXNY7yT&PGalS>%nGbKJ zT^rt(o^)Feo43B;t?P!Aym6|Hk-hRMa2FxE$P>twx1n23Txs*x54>&f@LIa}w8#_I zkQ0|TO~rb7;_}9Gr!MWz+;7irTK^xLxBlSmp=l|3(@c*)xl>DPH6E_@^>c0c6s(C= z{nbBKp=LH;t*PsqQ}VmKaTnB5270J1zv1mKFaN5A&D#L*wqj!q-k4v~R^5hE^Yu%4 zq&V|S9x3kLZy^k@6KYLPjYrxAYe^oLH##*LG#!@m7~b|hVB8TlZv(;Gj4rPyzo-0L z0;t(@(X?iU)>M zsTIzf@{9~`Gv9Ob_4fTX7`!#>u`~HS)zp%9Eybmr8o$QF{F2}0P4g*cW@=WUrYA&e za!MYTH||tg#cnm8lHc&Qdat?o3*gH4+YR9D`8g4IV{ch#&L>uWmp5kCRMdEwvtHWe zEsIV{(=fdK;M2oTuYQ`DQ2+q?}0Z!3Dg5&oXC2aRU`l4c%yq`2gvpIXlwGfSG8>0vxi z%^qCd_!*4YmErA{sXzXs&D)LOZO(!^@D^*2oT_imWZrzU<>}|HsU?bB-ZJXe@YdC-2~o7JXavzq={uQAD6d0tNYdp@#n)^sYAot%DpzfWb-x*ytVKDru&}qQ^T{C z;=a>h&a5?irY@fXBT^?CkF?e|p2Z|x-nb);mUdySHN36X5q9TK9gFoU!6r??~#?=ZzI9m z=;d|bEsJUUGVJo^n<0x`FlR_VTpO1+W@fh~ZQj$vwr_L(zQIx8t@BG&cuUAx^PSSH z)p)GE)U504cX{*8&{+KGcuR3chPO{Yz7zjoTlxQIjRtQo+|WG#_p}HnRZQMeoN~=i zVmjVZoRQ&e-8+7|xBZ@u0dH$oM&M1Fk`zy|mzq=c%~?JL>2E$MLg(y(I^9l5oo@!o zZ*^sOyJ*ojuC#d@3*M&fsKJ}{Yb;UDojIkwWD#dw_kQD7rRvs%t4>4mNO7qa)k;W6NpJd)x zC9o^Q+w7hfjW1-vy|U4u9FLmk+g7n7_@?b`$W+405>i=g$q1$JwA8}i{D&i`K+ z58fVstqN~Em%tvPaFs<5yLpQ~0mIvY$DRHM`+l1M-c}61F8Mvp;;f61Gy1fvezJ=2 z$#w6yjJh(s-E#b?U$S|d2;RouU1#1j-=$~4XFj}fchuYwpQBhgUEX44<|!0Ghx4Y~h2ia_eSYTrdhcZL)^c>q z@b|Pao|Ma*a;jC|%_6VOmAB2SKKyt4J)HvHp1v;vZ$+NA)Z1>}qEEowZ~yPUE52&; zHWj?x)uL6p`>n`#nx;2*+FV6VVx>4eMbo+7Zy!A8%kQvxn+Dz{k4wp0(WWYqAjMtY zingOz{p8BqsjoM8?zdaPTfYY*@W!86sCm|mOOvFBTJkeaKQ%qn#pI2hQ?qkl?efOV z)I2Y#N`l9{r>DGc(8uljZ8~^s*SdAO_f+bqvBZnXTNcxEdCQ_oDW~D>+|Bzre_mz= zczb?A72e3a)SJY)w3cz|Eb=oaYf-zr@vE8)^iWHA=#jLvmT_tFQcjmQW|K6t(Zjf< z4R06z?(etS_uEYHwrO1z-mE;0;#UBqQ;^~=Z?SeQRg~gVU0*-*aIMQ5zbdU_T%$|B zv&--{?AX0evU!^Y-j=j$Q{VSg+Ph|tz8==fjyK<~vi7^YWl^y-TfKIW-MmGgfZ^?u<3G5_zTf76w{DNrnKv?!r%s+WZ^AsgidBpK=X1Z6Iybx> z{;z$Hwt1Ti-nMsaTi^FI)`WfYB=hE*&DT#JeAoK=bLDOA@0b6?=4~E$TYGC2-twG= z@AP8%>&u&F7qLzuSKjtN`|Rd6Z}Y+1g2$@xrkO@8kMC3^zZ94H4Cjr%Np_cqv$Jqll>*{%925M#t(acHho0Ds)^}P93wETv* zUw&=0^YiH<@YZZo1m0MeJ7qjXGbgo_lOAgN^}KPFMoSenYqRj=%G+trty*f|Z;Qd( zvpw4D-_tCn!8@Efi(1U-@|I_Hc9ONujyG#!=6<{9^==z%-j;y3)pKj`#!oA?W=ix^ zYt~90-#q*i^jup$1x8qraY={xxmMDwD|xIFZ}?7_Ym<4SQ_@m(D{goj`qFV%+q^9W zZ%a1U;LXa}u%oQw(j-|| z@+LV0&(!et*_EsC+mYVp?aT^qOP5sPO;cCmoN?xn;?mw3XCB6>nP1ZMC-cS(TqVV& z4kbTxO4@g=~_qyI>y8RHa(dT0L)4W+~3jOCH19BP)(L+u^Ngg|~i#I;DG0r5@@PxAtE1 z`^jY9q+Lj7D*2_jrdxWbeRIa;Euj;`+oc1i;jc$}`+oaRg}2w1?;^ZOry#}C@g`*} zay%|?2|G2sefk^EZF26nvn#wk`jQ84{B%>Z2ga%C(P*xvme$fkE%}pq(`3+ir0R?_ z4>dg@+U1Q^s6(oh)*9XpdE?T74sYjFcv~~1bMfEPggdeo7?&O<=5Oq}dNyFQk`=7)g2Ymb2 zLoTfF_QLBmc;lyrTI!7+YWf+cria?sqoK&X-&lp3ok&{pGp^ChFZt<_G}q?CTd5Pn z+q6TkyTZBOE~@ahc|=$Jdn)ZtigVZ0lAm$OuVIOKB<=DhIi+|=C(`buTADcfVg6is z`+D2n$2q)RT;c7WdoturGo^ZY!g;G#=e~7w|NLtUEltEeD@YZ4B z+|N3^T~Xn!&v*~sq<$JpoOHSOTVvUUZ=Wu2z8RE$!`nW$_5X&$+m#jGc0T068_!GG zL&J>`;t#1St!-SqNxRTYHCNt#KkSW19Nzw0;q8$&J@oIXW_KZz(d3WiDW5`P={987 z+3{vg&)jc6`0_7Kad^9`!rPKbRd}=VG~7|rJI*+@G$nedCBG)F$wN>16c~}JNFK(i zL-I&#L*m)-#;!^eFuc9--b1@Nyj@-4ZN~a4yfKfj)=b_vXNX_=?37O-B(t=Z6QGv- zQkT*!Sa~GkW{~pI%ep?Zys0_3vpyt%f_$>Y=c4mp7gx zb)GxQV#49PDJNif``PAm-CuvZw!&NcX;pX&*>%WT<-VdY?DCdp^*n3k%G;@(r{9G4 zbi-SJzP=g$K8G!jRsk;0aY(g7z}2kX^?>89sdEMHs7o(gRRONdTXXPscjsRE2UTJt zR-9AHI_Iv(n>4X>yzwlkr7ESkc~JlG+Mhpa^VR~qO`B1JH|bj$DW1?z>W46kTIqOW zC)8Q&IwU81OUGM^>V~(DANbxeHgDH~w_#7DbsI^MG05xb?<)Rp|{ zcw;u|giZ`^huwGidu-lXg127Xdl%;IxcSsyMF6E{qu+mZQfdgw&~U!Kapt7< z%_I5Cr@+kA(yGQ7*Yg&*JHy+zetPp$Hg9dg+b#2J@aB68zW$IsTHil-v0Xlekm|l` zIT>o-JQ_bIlO1o?E(~v1{rafy+Pt*`Z#|x^!JB4xz8+44YZ;d`{l1zx%csByvvHF2 zP}3hvGe31mewR09mb9iKr&RJA-hTMQgCDhdYY*O9_v@SPJ=OG61P|xQe)7NHSUp5@ zM;h(&7E(Q7t>JCz!sE`jdFue)wk_Jld6VYj^2W21R{6%Q8^G(u_pK%Q^}KNc)KX?E zZg^|f@;9?=-a3M}bz5`djr~)zXU3`NiKSiMVr6EYdhy2TxxDeK)a*QkHoWcl_t!sZ z^VSKxtr*yE*LzPnLu$^jd1-@oV%y~{qa7LEzWS#@zqWbn4Bn(L(kUeKCap@*x#4Z>gC`$s^VS8t^?xw}Zz+0);MZe_H#Mv7)*RobR-k5=! zRV;0ITlwtq=h?h<18-Yb)Zk6pkyR`8eWl5-@%Z+W%p0?5s%t#VnWs+2TZ#@1ZzKD3 z{eaC|ckp)C%QbjQQ8zVvdDhbNmSQy_f9RXWe??3(5#||TB^c0D>Ckzhij#r^iaFJ@v9-4yC`YHTh|}F ziN8Rk%v&Guw)L)@c$46U;O$c3 zFSWF4HN0z#*NY`h-SR1DoZL}k)VcDu;=IR4+TXwR18*}%UGKi9yeC5@?3+h(`jW@D zR{0dTO46DMa4r3eOP$kC57*K|EqNH%^Tt)IqSWSo+k3)c@3ndB58g)ISA{q3nza}Y z?5*m3$XZ$JeC{_+OtYhqsj?Qep0~gb4R6!eK8L@cy!`#!0Pxni#US^7)6~t=!|BDU zTRsKfY*Hsu+&6y~{O*hzs2TlFnAj} zp$c#8XIIrOZ`Snc{u52C$tUv`t4b-4;jP!Qy~f+T-2mR&t;>lwem_7Ralc-KxV#mi zt1PCGD{qVXKl-4}+Ys=!t?do&d#c&B?``1g*YptLae0%n`F8H>m+Hof=gQm9uDWtR zo429h?Y_x1cq`iULi*uaJ#RdBW?}-qRMXp1I5B?MCo6bwdr_ z*tte4d-HnxG3EZ>`0iTc*Lb9U7!T)-*%GwjZQgs%KGEjwCh*q3;}G{f<##gF z3AJkS!#t9Xm0y!b&zoe)svEY~F^0x2L-db>CCgDuT8? zM>X$KP0!_1&^Q~r$K@@f9U0!ZPUtTp%B2M(HYkk_TVk#eCxH=nn}ZF zWY^$LGgX&2&8ie0!`p|po%1`Jw~65GvFAN_lg=(hZw=>OueG~*i#`Fv+yAxM)A|3W zlfc`G>u=V-r}bhZi@MRLUH4CrG^OnBH|zc}yp4TipD)_?+hp)Iak&R?R{p9JriWUZ zM!n+M@fNb{khRhzv*XRGXn4E$U*rF4^EL&%^?b>LH|FPLsOiZqO1 z;<@s6`NC#HZQiDWx0i1iu76K8Ki6D?zfXyNYRRKH5w0ayj7xdgL-`aKkyc4@u9d3O zBjsV-*Uz==fjU>-Zaa44)i!U_z}rhJJ$U2p8oO>>_ZvT1zLQVpEhMwmiFr@&-*5C> zo3~rR+e14t$@OEH_!>_Y> zn*rV?tu~q zJvOl9nGpC&*4T zevL=U>GGz@Mvw9nWO%!#=OKM<-e!ZhC-2_HdE?xug) z(uTLYxBSBS`?%(Uw*{j|g}2Thp%vRj+)_L~7 zZm{pSdEjmM{SkPxIB$40*bhCLI6b~vQ_JN|TE*)0P-}RjpE-TClt+rYywOSR+X?-p zJchTcW*$Go=50QByZ*Y-$@d%kp=K?O4q40mQXa-7%{=8(kerNj6?KTmcP;(Y>_^hc zym1vXQ`19DPe~iz_B-NO=imES0N(oFk`r(2lbSs<9-=il-TN)1V!hUKr;>Jg({w9k zW8Avc4R3cnzWy@%ep?9M+CNl-H|C_)^ju_5cD&^|B~7hl-ZUAM=V*9a^Shqs+Pp0S zZ_l(Ill-14YgPT8EsM3W=CfN|O9RxWRv%u=?j*M{T|=Z)19CSZ75apL}0*t{(RZ?8_-#d+hb zsMGPr43g%FP}9#iH9fI(T;3A8GQ8cncI}63-j;*6b&o{mjXy6Z^j7oxD`{=4)5(rE zcFt*NG(5tsTff(yr+)sl(@z=SeMOgPCzZ~8Wv6k; z&na8uFjBPBfEaT!591omwbWcI#mk4l9Z9R0ftr45dZ>LpzJ95ei<{(RMQZw~Esw$N z@DKfYa0RzL%DkOX;cdzE8oV)o(KKoC?JfLz3#sngE&aZAlX+w2Ky7%t{E{zBba*?n z!rQ3F8!>Mo^U?FBIp0{GkR3^DbLH)`T}OV&;q9ynZ=Ji{lKh@ZyJ+ljJ#Sg>^iuo* zN^(oB=|S@Ad6P0|s+T*>l79c#@wvsU^P@*Yn0I z)XdB{H9fvs%A<+XS;hPOB0ocb$=w{t4I&7N0c>J(iW-g^J}#s?hU&aLn^_L(ZYu@-xvmOPA0nt3FxSxb+U zM~atEfnQ~|BIuCbxYp&3U-jK7*IL@}He}4h_}}$!|N7f`72byToluNgoOvXz z$wLqGFi!3A##PiIpNLXL!`roAJG!02+XWTgUfoiKx4^nJzX#(q>ZOx;t5;W=y1u95 z@}{Yh!ee-A{ryS!<1%l*rx#Xu+c03F`=0vlC`E5|XR`6~<~!|NdE4`!KReR7-!7`~ zwsdI~-lQFoLu$!iv^aa-&6_&`&PUP-XKL=Zl?R_S+~Muw3U4E~dGMA{%j>@4UYOH2 z&b7YUowjd=#^TrCZ_@mgJ2kw$zvWBw9NsRe@YZSYB=$(2tedq?$6LL+ z4Vi|Xw~#8OwT8DnkN(av4sVxMc-wkg4c>fvXe=j^jyGun-rjfoJ5F_Y zyR5?7z1t)6CY?yuacL$}JT7lp?=B>NI^LwYC3IqVJABdX103Eiukbcw=;Y-0G$DW0 zw=$NWzb@pPQ(DUz>UrZT-%jY4x?)_?%%jolIhi+Blrl>mD{gq(wtd-W9p0{}@HXQ1 zg7PNqJyu-Lo2DYqSL4sqlgwMJPD&Fny#4r$Bbzw9U0LC+)2k_YEAoUjGmOidredBR zo=!fzm3C)%JK$%_TRXh{x5C@DVN;6vo@RYI`SV5wvfdr@=fm6E_uIRFGIW*0+f^0b z*4(+9x2E~CM;5u?!#Pp`^a;x+k&?ZZ~giDX88LYo^3Wwf5BP5;8Jzm8V|?k>({K6axz{%21X>U zsYnmCCa1<90_4X{A-`C09WR%Ie6PR#)CK3m9(a3X)WWLJQ@#k zQkPGG5lOQOHT_at^3b1ff>A!vdfudN8BfrLx4{Rs`Jl~P3-EUDn#jB*^iy?udHQ*> zdfxKvFpFC0cuUcV;ca90j_!ClX~9GTl5JS-cD|E#|oRb zmf&sZ_!_)P=hfKb(WhPWlU1)zZm#!R=QVqGw|Q#?-rBD9;EmsP6oEHQKhi0?ylJw9 z?3($hLvm`?=EGZQN9KOp=cSJxZS&R|yuH+Bdh&a!*->NkxV$ySU1)mFhqqEEhPU5r z>ow2jtqpivH)$8=O@F^>&aN?fa^>xdm##X?=B+JwTeN-`=S?DH_x%=q0_J}E=h^%H z(&nulcpKAUhWnoKy9;WbofM~sTJkfl(acFLt)++Boi;PjNllNhmhx!g^hjFjC*A$V zY^*57OCH199W#%*z~-$zcpEh>C2#DZXj+;H<9gn>idwVl5Wlom6Q?H~Z_@6V+430P z4*$fK`)%GjfVUeqR^d(hp2x~x^H%`aTdbVEJxG3+H#*rlCOb_xyE@bH8;2Z(X{~bl+3v37KBVTJ|96ko?KKNfmwLAszbGlKgt!m?1^y zhPU5e_V2Ow{niP*JwKxcZ|tE6I>j8S-oyFuR)mSA<1Iz!hPQFsN3^$j>kQu3JW++W z6nAOEnR2bBXY%8_mVV~Z)J^7%nWbzRkF-`3kCi`}H&&r$hm2FxQ__aFwV!>}`uFQU ze_9vtwzB&y_dV5|9qUqCJv4C0oYeG7c_csAN}4&TlX;V}G0tqh9s2rNS6a({=y!RO zoKl?G613s%sNl!Q0MW zv)%WU-R`Pdf4?>M3FOM##!-J+X7knqygfcYC*ERxqGRQ0yuA5NK+l_RX6ut^-qYr* zzTeO0ttWU}_H0hPS-quxXE_sU-~8!#3#sVaVJyF%x4?>qx8Jn?#E~{{y};Ypesdbv zdm8J+!+G=V#Md8q`VF7xbiAeL*6_CecLP3W^VS=@bzK~pH~vg5Y2Qgoc^LPdPdeU0 zI`>^mzqC6}fPTi)@y2XQZFpP#fn%Isf9nI@-q;$MH)U@PpT2te^Wm*tyJOv4d3)Ee zJ)f}ex4z)*@qu&o?%V;5=B+cUHt^=#TXwwpcH+BJU%$(nrbCU#>csHY`GL)=ZQia2 zZ!LFt@Mh(2IO{y)DdteM&+;k6%9iH@d~3P9F+;=&7~Wd^^dB>A-UflUO*hUj{(H)q zP;+j+I{AK!RmHa}ji0?qnt76Wqccz&-roP0b+c^V27|ZT?(pD^?>n{Br^dszzJ9Lt z)#X!QMQYz{vHVgGzH#=GjyI_yGh5p5cF*gFea+_W2JklPwJN-6YDpf(sjc3szHu6l zl#_94=49OEjZR5313gll9%|l8j7xc>IM+%!^}MkPHLE0O!`swwtREd$@cv=6ukAgyAHg?nq)|x@+pL5 z2w7{LRP;|1dvkf?S0$aIL&MwSgD!CX|H6&nZO0oPys-yrsplfZr9SJ+Tb`3i=8f~A zW`|Ooo{~1aJ@evW7uxsRP2lapQ46zsPuVB6)n~(}TjP<|`o`n(rs-Vc@!f^S@A8&W zCx*A*t^DVSHgChg+pPPl@TTcUzZy{+-kw`|;WnGM zo59=Q7K`HEZ&^$%;!GOu(tLQ+Ohe-dnLw_*{d8%Mdu-l@gSYF(MdmGJGSXVjiD*2` zpUj(YHfD~cH9crNx$?H<``>-Q=4}Lcd*XqVylHkCYx0^rzWpTgrm5IiJT7m(vr_KV zyreNUy{G{1i6>yiALc=;4Kd1@(xv^z~)%E`LSBl+pkXqPuB zv!)YiZK)H(+e6=faE{H}DDXCOLQ3AG-ZT>kS*yvf@%%5}Z_<2x&&Bc^-tO=4%e6Ld zqruytbt!q1dahU8+Iz##1?-2P&1Bv**_b&`?c1AsziGPVtTg_T$MCjb|UIhlkAwbW96 zdXjnb?SvJrwdOrN_m!_ZKc7wjZ~Y%j$s2dnSTyI8&;7=V^_o&LZ|o|tTf{pPI=_YnjJav#ut8KKGlZ6L#*qYst^Lx$<`YgeC)R-X?>$yPovmP1;>! zjOW7}CzAzN%r9w|w~RV5_uKs&CVs}|Z3=k1wa0S(d&*CZv^yy-?NN$Ld3@vSnK~VB z%phs0L*Jb;KXWopO%Jup8^0=PW~P?>rMThkiyJ4u)8=g|cpE;~gE!WarYFU@R-@7*;~VR+P;42v!;pDla4oL z@a>#_i9=0(dZ?M7IxcVQoVrxe@YZ}-{~K)HZUt{I_r5LpJ+*SC=7bX{f=)n!79c4R05nv+Zd6 zewzv2CiY+9zNb3CyT)XXetUqAD3t!5hZ>v`iUYIc&K4R0U(+IHvvFU$pR>z3ExEg^r+ zGcEGkbi5ULM^bNk-ZUgjRZ4Ng+rqyebd~*{&I50YUh?2g!@Sh9W^KY=sibi5^SCEXi_w~hC0{)x@o zLh!a@M@rsO^wZF1)~}VOxAH0YcB0Ab%ZAj0R7;9Wb<^=CWlM+~-aa$-@If|ji@@8u zn{IdC(}bK2%rth-?)xqJ1PpK8|M{2yvw2$#-WIL);LZ1wRb(d1)83VKB<&^L{U&9T z;-#}Ty!GGuo_%cImVmdLU$4PiLjLG?om5MThwz;I>Atk@CDC^m(rw7vbiAdgXn5N+ z=3wW~XDtP9?MB@3zw@4w+hpE2ZLVUxvFKcRTeDZ&-`n@wGVr$L?v%XoyIg9X0^^d_ z`00tIlX=rH#vR4#iaDt@`K6x2d1EJ%E>$=8+cn3Z@GG0Q<=}0@n<;tY1f-qPQ#9@J z7Hb!=^2C~$%bRb9O26Ul_PLuzR(M;#*Pf?-;k46F8Q*9{kjl~m}w>)=kopQsc zSc>a;V+LyF?hJ45Y%=0|4sT~wczb$WPQ0b)zhRQf{WkbDwbJpX$?V%L{Zf7@&b5-( z^Co3b#tm=(7=QWw4sT5>ysdn&25*|3#`0+PrK!a{l0O}9%q(fCqU6sy?()VCsRKJP zynXt#EAY=Ry#4*#e=59t2lm-;FJ+Na>dobi)v2WndB&xFq&RzXdE-}O z=@Pewx0CjN6#u;Q+q|7!;jQ3EYeC+ynr z_JMW>zT&*6=TvxW{csiDG>oyXV@vj^&M z-jtmf-adcoH`h75om=5;Tf4i{y{A$S+^I%05B(u=&00Ng%tkGBOAociukkRaq?yO% zjbEjfvN10COL4>7h*3)(a(Fwh!rSI4Rd|zfvaY1LmRj;_;>;sy-#q10U^XcOJ=Bt4 zigPWsZ4sRD!cw5owp1AL+)Uz~6enzOJJQ|O*mT{LiO=gXUReb%@ zTIP|oG+~!FW?)5Xjfeh{HoX1l-7ODvc)PH|+pOtTc#}G}@;CfCM%HoPel&iUx2$)e z$*<{L%EPtPE^ktXz_{UUQH%fQ@^(>$x9cCT!dqb7+&B#BvzWYv>_}RhD{u4WeE9(9 ze!IBBTgz_uhQFu$G)cRb;?i0v&ODMw^A;rY#tc#wc0$dyj8kho^oR7|^2X}atfV5~7Pk|k3b|HDPj)zRo<&E8HGHX1goQAg> z?|q<+!`r15-tONNfj7;magV;5`Kf*L(4Tz2#j3)JlFp(QbGp3ot5R1|JYlWjZNU|% z-s|voS%tT$J?{&DPo>@%Pw2DiUFYj(Esaj*O_QOqcwF8x>fG=)ZtighJG@<9;cei& zl)Pzp(0HW1`>vJzQapd&q)s@2SRUro^A=d$@b>gCK6tIe+Z7ewUfJxyo2DP`w8%Ox zZ$+L=GH;sBHJ;L08{X#sr7Uu?H|FQu%BR2xol*u( zoE~b{Vw`p9VVr(yjVGBmW}{|iYI-bfc)P#twLfs~xBpgnTd}YPZ&uE#-)qssiBZ#2 z1ReK&E5eT011F!3H)d1*;cvs+x%(ggU5B@;D!h%|k|A&Gxd_@l1x;66rSTWd6V97* zM~1iW-Tq+D3 zeAjYcdfvFow_?d}cx(5_neNXE*H(Dj@j?yW>NT;f>$>+_))iBnV9oEAH91}0LUt;x zEydsB?YFH?#QnD6tv_Gi41b@)gM%JO_n>OFs_{r0(5wydxWI*Elh)R2ckIUnj$e(a z4RH5%|IdFbz?FGx4&LV8Rv_M_(^39_>&=9daVf&q*>)O%FADD4zmXQF9gJ)bxaC=9F|uewR0?if^1UsKI-ZH9gc>BwT-#f$RttEJCd3z1sLe4bS zzCwDC))t31sYBoCNqO?&tyJCccFCEC{M+WO6?l8%l^VQhb}j8RWNk=JO7!+Z~va!{2-gR*5GaVu!oZ0(=1Mb-zRXwj8jV<#(lNRo0LK7M4F7`m*T9e z$*IXhPdIPPCh5{H3~%55`N4d0dx|+DJhkjoz^~OAU-ndGWnSN@r z#5~mW$I{`vu@iPnUGf;-4mkGH-?n*c3*JWVOvxL2kTfSuj})gTYn_g_th@5fFHMs1 zxV$BFXm~q+{T1)Gd20vWI*nYL{GKM{ulXxN+;vEvEY|KCylEzrD{t@H0q#!q^#y#2n#7w{LVmjC}k2k`b_vvuk2H|~hKUi`4Grk`D#H%=oi zZwa$C@9AM9J8ZY_w~pX#&X|Nb3*q%I^|!dk=Ib5|bvBb&EQ z;BDv{58n9mh%`xh*qaomCkrjrEze*2YDgziPAM+c^7S(h*OD}M*K{(2Mq6tQZ+Cxw zwDbS#JA=2LEgw$zo@)A`hrLOferm}t#ig~3mnYANq*;X?DNYYH{Tj`+)LhHBq+Q-5 zXJFj$HtdvNudwg8F5s=h_?&nP+)K@$g?;lU^CsQizEz6g&y}}P?_YbD&0AOS_WVOB zd5d)-vGVwyZ21&4)nj>lJJk3!y`|$#lbIcs{N{eU?TIgSvw7B-in}; z@3$g!7^??8ZyLrDJchRw>(4yW=B+z;n>uMXZ%ub?_Vm035^i`~`msJw+Pw7uZ`ZFc z5N`>eon83*x2$)a%v;3jhPUn~zGnS<;$JweCwObtVMBKBsb()4Psry^${&8GNtr{s z&0=jbZ&Jk+yEDACS+VV1_WjliyltGCA#c)NiVzRytq8l&^pK7>O=inucsuUVwa)Jk z=?&giJenbI{ESdreMWyPHTfGCZz0ojc?-#wFk8dh;U78>e<6PP|DW~&Z{s>YlHGfn z&|6&?V?TP{G^eBSu)43GYpFGPT;7<0T9b`_OB>z}*l+X6_IuhFymg&XgSU_#Le^S) ziOy#-Z&`O9lAmiO&FSfR<0_42^^)K4cIpKuf7s@&A9#E2@fy6buJ8Xz30WIbORCG9 z>9kWR;)gyJW`xJ#L_Nrc~+Npq3MAh zYAL77o8;7F<66sacpKAi!LM!J278Eyi%VI~Y7?M9EkDj-X z4ohndZ<`K1!ufgOdhph)*T&@cls!n_y@X7I`6bP|l9tvo&OVcQV-;rb)vV~dR^w-% z)LiTG7ON{(Eaf!3t($$%2>X5;1l~5!^WcrU$Wu%6k>Z-(l6li)2=Rn;?pxR8jTs_# zVt8A>{0Cia-UfrW+n=ezTf`n3-n%Sva&Erb<*jHP`tC^bC-Ww)N)b1_ZR@)1Q#Nlm zfVW9~A9LT+B2PY6Z!T}X6OPr1Z(U!%%Ui6@tvqji|2A#Fu{Li*z+10HRe1B=QDgL% zPr;g4!)=`Pj$+l-|K*H{C?9^DS}7O zTi|)Ub-(Sib&Ac~jo|I!0gt=yX{;#)_Fwb13CUkf-a>XHotmCEDT6X@c>B%r&iDsE z%KyLqCh)d!X@e@T7I|FWvN-E3I`_?)D{u2#{mA+IEr)@(QQKG}GlS}t$CnH!5gnKx-ZDRyV>x9k7<$P4y+dNX+IF!+h=-qRF4SH0^Z&ZHrKIB!{W zB~2`(Rx)o=#T0SF+rY@}^l8tGXsnEKjbyef;(7 z{$}$w0=(V1y#{ZwP9fIxV&!ppipYj7jl?P7H6YC%!V$=4}*s8*uw>-Xi?@(-f!E@SSAB2}bcG^A@qX z;cd~S{npvMjRtQmUiIKD>#1g)pR;xEw~*>wD`{4zrl0XxIvsB*x-z`|aQpj?w0Ro? z-X0mYsW|Vcw8!HAdqyE_NV^Wn$+g+>W_50Oo7!*f7Mr)R;BCR3b>_|LIm!#dX-nF@ z-y~;akEi1;#jXu+bM8L)G@G|^;BCatI`fudUs0IU^Co>Fq7-`1#;DWr#tCpr2@^29{p1h7JJr75CV;mM&7Lmi{ifNY z#v{$AnD<-Aj*@pu>Lh1ib;H}3XMXJbe#?pAZQ+5(-38qMCAlWR#@S@WyZnry6MX~WyhzSq5I-*1z^+p;xP zc(d|UWuvisl6fog?tE_{m$#7JB~&-O^}Y5t6KvilgST-lH;2Ec{ESeC?2CD*LvnJh z%Uc#zSTSoYkr0=+gk2ck7X1C*%WU4JfVckRBk;x^vOYB_za~ykGH-NpchvMy(-WdY za&m24-k6!YE5!|O4WZu|A7Fzlg`_^)K^UV;;AF?CO+H|~0nWcDX z8iuzeANl-H`+l1S-nO=RCj32(HTgXA6og10r3s_>S; zu-DHHdFJu0l@D*eUGbDd)-q47ydAoHvGe!$P6ux@)>q+;-NxEAa~46nycKz;A-(0w z+tA~Cx3TZH8Q^VjhiBdQl;0CllM~-K*HTM)=%J>cnjVdIdDE-ZB;k#>35&~lJoEF&IE70rd8q1%2V?VWe<|pc;g$*T{BKik4AGXHP3QQSY1fQXOViMJq_w_rJ#U(>5~g8z zd*J>azqNT=2;T1LwKeX0%AVPqW=dR3ty$~qDW8H=MT$#1lKkvSvsRObIVH_J$-L1? zEoEk${*pGlUB2ew!8UJ;z}u4fHF#rAX=3bwnttY?hngO0dZ_84rbp7rym6J3nY)v; zCJ#MXXqPwch?>>8BgPZ7;q9@lM_gj_wivukdbS2{3Heih?^J}G2CoZoM`6uj+R z6q&cIIg$ABty`Y9?<%P};~_ai*0KlcboU!GD7E43qCS`HXWwtjz}tqck$F@0R`Y(# zA`ee9nKwFpI|*6qTTA25mA9Y%YsV8dZ_B~k!htX3^PXxZqw&N#&GIQ|DoUPMok;m( z#dGED6Ng=UQH8hld+mAZ7f(C=l=0nHbeVQi>5dt(ECOyt7`$%}$=6$~qn5go;;gIj z&@bs+aU1xMrCm$$kepo0{8F4A{q@Fd5(A9WPtEGo z^iVS={l41eja8UUiqn&z&9`rh2Yl*Mhqp5;yuCSiTmH`}_eCx3MVh=6m+~;K=Z&kV zL#i_mb*!As%9Z{Ud^1aaO+{%fj|I=v;aG^QU^^f7idw+j$kKO;#^Bjzoa=~ zdYC7c4(E;CN?l32V4PaY6B1A6P4iD%C_6N~&Fb&~{&)S`_uB;(-a3rhp8TG2r>sj& zPZnBpvaGA=Eu1%2%tHI_nmurBGH+bvtF1X3-kzB8hkrQt+l3Y0Uc9#oZ|qj;LF19u zN^wn2Ymcel_xyR&>`s~tcTG({dL&QJvJ};taHCz zR^hGPgp|B-M~zV@^QM__W7%~$Z_-Y!DH-10{L#DKba=bG!rPX$jhHuUzcs&S)IU-Z&y@!yQl3d>E6?jQ&9Hr#gA`4yEbo}a5!(u*&5#7xcvG@ z9p0|2@HS&|4c^$fR9EAX*48T?mp4v;+IM#$Yo%Jb^7fk(UtQwx_TLI`12@#*P1;3B zJd3IN_M_*m2;KVbj{aPETYl5NKX-V$s={0Ij;|)ar<$LJG#*aCS2I6#NFJ9ra^+h^ z-L=abI|6dzATw2RKE^odWm@PpY-ah%opUro8ySBpHy5D&Wr$DNYk;_ zTAOe9Fi81b;CwSknWcEv`K7vXflJth0dCE+&puVT;L5x;2XFnJtTS&3{nunBWUb4a zbY4Y^`|dg}Zwb>dyj|SyXzTAA{NiaXz}wF5uO~mKzPn54r-7d=a%$?*(|CE~E;L%t zo90{+JchTQJ>I*UeZO4?-X57#gE!5NG@gV$8#t8maIK`JwNkt|ym3cTw;GSMR?nN1 zK^Zr^-MQd|uh_h`1aAwU_TVjK*UFx2vKK2)I^KLI;OmdIJLbuiw=Mhp>)SSOt-#xe z-aC`uQ|U);#yM5Sx$7)6>-y#|p8_+)>YO=!yP`iAH>n@r^Y!&hbz{Yod5cx02zm6p1)h=NZRqNyLu}sKg15)}y^;K$#+q2* zUcA1q)709{Tl5JS-ujF^_-LEAcHr%{#Z`FIoR`EQUj%B&pGBN?saY#|+R`d!Xe?TP zzXk5d@b;}2zVv;YxAx#|%JWrtV_j+Tj8kjYhVU${m3ol+DW8I7m6U;TYE4dhq#gO@ zq@OyOH+I5RDYW5j_1}-3ZS&RvybZhl&FtP&%_LLotKn@8$)9|`g=7xdk?&efpUJ%W zR!s04-g*SfrEGNkYr-iFp+{g&fx9A z8=5ugdr89!)p$>V6XvX>c-);zRoF?rbgsPpYtl=%*}QcDZ*x}G;4RNd=2-Z9eVtvh&o>W;{~NxRNy zk5w7d)Y|oUlXgwg((PJ#riQm2gZ@^*?d{)B-UGbd@mgfwq+J&+o^J9*+l8i|aNaak zN*=@8%(3&|Q30;}`^kHPx2eNhGzorBHNRG)>^b#)#U7;3BR%C)kepJt%;4+Mto6-F zzm(tQjZSK2^VOw1hPRf79rPuew_f0F_}x`_V-J#MKlEr$kbdT&)_8pV%<1yRuZCpi z1ejCOtR-o#rH64zySy=jq?t|fSaHMK#$f~J*u3=yZ-d{g!kd*R^_}CJhbQH`*4JM? z1>aST(eLt>(VPu$2ekk3D4VxF;H}l@>zV|=ry2F1Dd+#oyk+#MF}!{Fs2}#TdFu<_ zHs2S4w~YFa&Kuu|e0XC;>X3=~=5g=0km?C*4R2%q_1~s8Z~egA{ViH1-*4OmY$>yy;czbYM72fzh zO8M!r>Nea<(pvhZJX}l7wIRBE3Q`7Eq}KT9(X6Fknuf23bzR<=jhd?%r%ve5@b+wv zfnT+G8vx!GKUjq~=HFFyGH(13z+xOe`;H}HVl)P!~UWr4lrI!4R(@!mV7^jXq zZDx+8rQK=rb8UI2{I4cM3XkD!)f>M*%;s$nczbhQO5RfRQ^5Nz0>qkMH}}<80mrgSSW9wQiEjdm6Zxs;q|OOrCt!EJ^vL9Z7L1594~?xJuFqofzKk z>vxIs=bdi=Z;Pfx;4Ptts+>rB$usWq#$8aeD=AJ-(R5ti5_V^Jn|bjoQ|RfsI%J5H}Z}WB|czf;El)TmJQyio=Z%@U3&yE69?37QWn9w9ylE;* znXR>kx7$B))Gut_ZU%2NXI0@%${#DvKB=?LAD1_F%UwvCern0jIQ^2QpIXlwSNT>c z`3-O1JoC8k*t`t~Zxc49<&>S*!8rdDCP|@EG1+ zTK(`io41kRt@XSbyh-_`$uKTyDJSF7T1{M&r+f;`##LEp)}qe3R<68V{kOg&Y~DtJ zx1F0)@|MMv^33VpZ@jytPBb2AZ5DA&Kl$)h>cre{4}9&*SK7Rd25*n_Y2PIAJ>``6 zsgyMRMb<8Fnr^vkjXzJ1)LTBhm3C@)``$nM&b4_P1Kw_5=)qf_(_<~^Q!K@~R?_8D z2+6>;)U3!jH9cADTzQ+>^cT07)praNd-=czAg@ zoYdu0(5zx7)XYY$@z9^OPRAQ}Ax+7OoBM6Y8#`OuyiEjeBX6t1o9_gyx>fJAOF^}7 z{w)0EQ^+E75pwcG((#sJ7lyZw&N^nD&D$jK)@gf6-ngR_y;Ws0&peHbw~%S4<1Ix+ z!`tcC_Pf~TZ8CU!en_V#xOs|s3MuwpljmaYG>cDMI^I%rVtCtg-;K*`-ll-J`&ZTA zjqkXmrQRez<6O(QMsuyNc6npPScFSC8E5BGoF0v49+x+Mm0HRyc^EHg!`s?_4C!j~ zHWj?Bex(L)QV&Ilhfv}2R)ig8(SyqytAxzZT5EWl_p<|?KVLr$yiFO_xrzEcwer_| zvwZ!WLo6-zP(B6etFfxb$`jIOGH>iqQ$_NW;)b_lyZv#3eZSob-g@5Y!5e$?)z;nH zz$)WXPI{>6m*zkZb@>z+@zv~v+BZ)uKYO5ddE-|#)iwT-$ME*OkM@1H&D(VF)@El7 z-ZbYJ%OmYQi~PyFX)5N~tyD`BcX`vSO5riQ^}4v-8#Zq2K zMbOE-6`{jeJ><&UXAYfkgw5Mb@V4rnl)P!CQ7`s1b&JCryQS7lSmQ~@o2C=XWA3-7 z{`u3*HgB`Q+qh<3o20tmG&?P_r#QT6revLL!*{=$wdr`%WVSqpx3PD<_fVU++2F0$ zm>RtCdqwJy39wJ*WIRMOCpB~Gd1D4@X47b{rPizs@#M-|msLxf+q}&IZ!Oo<;4NgQ z(ps|0$x};fr8sj+9>(>&ah0#;j;MX}g!o)Z};6&g}0D?GWzUzv8U<5 zO_i{x-z_d<;EX1vG2FV;O)8gJ(}e5o^ndkXGDr?*3zTV$-HSg z(NrmdCmnAoIybx>`h^ysws~6u-tL@QXWpcfmEtMpQ1!lTT)c%$BOPy2MJZl77sK23 zi$D5vo42LlZQ`Smc}wW0>bb|_$2U(nZ$;ZF`=L(98|SI~vrdM$c2Bb zJ)0!Hr|g_P#L~=99V@?-Cz&^CRng*_iG}l~+_mBDyH9^_jLq9}@b>cb8oViM)&1_$ zOrCp5=FK-7GZ#%~$D1`J!&{d>{QLh^cw4{Mo~Iss+Uci^@4lkTw3AAVJoI=SaATVZ zKkhUjEP153=D=Otm`&=C9;u4PBdujz$|Lp0wbaSDNmVp)s}qCU*dA?9so=IpnYS}4 zyshZgt4ZQ>%Fdae@y4iK-iok0O%ECmPb8T)Is>)g?ZnGJ_9ut8Gb_A}n_Y#sYd|11 znWtu-j8oI2(b8JR*&B1x!#K6e8^6j7tU^r>*T&M!PwksOSKbc)$VX3hcsr}Y+vumN z@aDTC-%p&c-?tx)Kbbd8hQ{K_mA8w2(dS`@x26@|y7lVa1h-k^It5>D>DTX+tFmA$ z>-^!oDLXXp>CS_it#^3)PldN_^E`NyZgr_2`l&-sjd@s?apq^7{$$>`irJV!6PNOE zZHR|!shMBT8#73nnJsO2`|ak(-{bIhc7?acpQ*wdze}OE@;ChaNDnpr>|csY9{OYH z@+risDCOZasAKIylSj{+CY!X=(ptmYE2s51#^LRp3UBNC_G!{+-&1Kmnp>!R3Q~qV z<5IWmLF1vHIxcU_Y-z*Ww|+Epy2IPK72XysDiCiWQ?>TqaQ8~_;_xPQ7%QGDZyjH3 z{xgTS^D4ZJ*jgaoVog|+=YN^Ej6T=qe%o~PJp6Xcw|~9&{0eVf2KH@|_@4UqMnAR0 zfW{-O4T&f7R+RqkV-;GA)OX_PeQ06lqB@}?P2G7 z_kGRm_3Smz%shMVn18r_*Lr`m)|xfQ(&6qmZv#HF0Ran|Bmmp3|nH8Wdl4R6cF{<*Ki+o@IF<_{fMa^6$s z4AJa|c^HqSnV(wv{8mnZPIe;2>7my6*@@(*hxw_cwe(QCylFB>9w}a~Xn4E+#@;VF zyq#9%ZTj*Wys=wq%{1t5W?eajX6{hao90AZ-k4dFf&L6_czb5ue$P3)onGZ_&?_Fi zWz?$sJy^tj)vcxJvw3*aOh)68cAd?ev?@p3@OH|8_5bDY)~?FiJ0mVHIq#`&t&nN> z){5nC9^PW@NXkaQRrpUO<8#~u%sat8S zub+8**U}%(o79OFH}~5=P!??>E zze>%D)bwa{$XY2sb8@GQyS(wM)KV2?-0*hgU;3?Zcsr}g+pVJpl``K`?n0`o@knbM z6|bB^NarDIi`=!#TdeBV1PpIS|M_cwba*?v%G;zn>hKn;w`R*zIfWv-vQEeAMeyyD zwbJosb!B+_#b<`$uT$Oh{o6TJ-uk}X0NywatLLPrz*@!g#_GO0D4PmTF0H=68AHS0ie}+h^|>HOArX z{3>tDCe-0ARzJSn`uctQbb0g5P^3R(8in%q_J>bh2;Nq``{(Q1_SLXwc)g^6pVlKE}f4l3fDsUCvI)Jya z7Y!+8KB&rCUcazpPOE3HJRxf}fpGJ*-)q zvB$O^G#>U)IR!?jHQ7>h?%M#^nKynHLCt;nPTSY7sio&l+6Ak`(jhx#59~QEZ_G?xu5Nhy%>UbeXPdV!;BEDq z2)t>g#J}TOd zN9CM`w{!3K#7LXB?%=J@lM#5!@mZ+JOsqVbc~<7niqx76t>PhR6|Lz!$21IYz0Usr zXKmhkfVY=>k0`0%(?osNWRJC^y-RVfm9&2HQifKE=i@EM?hJ4DzA<1Yo420e?ZG*b zdCSpfUEWgU)bkcXb&Ae?a~8_m#ZNtPr_Eb0@HT&K4c?>)OL5-`G=pEyTk398Pqk3q z4&LF3J#5}C0dJ%FU0I6yo~AgR6ep68H+DkZ%)2g>w?7^^bGXghrQogW{1)IX#cAf_ zEk!3%P7Ti*Ph8$IDw^-#KCtJt-?n+{4c?x6b_;Jce*aWrAS9k~Qc1sq)$VAhqUD5Eo4Wm9+Jm*t;?Hl2BqKd_SpVwe_`|17rb4) zz6NiNnv8N^QC^?oc`I^vtd-4M$XQ!!4R5o*QaZ@ytsi*nb;Zb1uJ@EX(rBx(ljg&{ zD!)^?$VnZ&seR@4I64*$jE|@zxBT`}UBIH>(rF+j|b)y}!-dVDR?#m7`1Q z_jGgOuo&K?iAizm1fzV?3+1hJ^p5xo@hg8m>k9Dp=xrXng-j#I_n*0^%|9!_y3{Uj zMKX}Omb+6@$8(I>iMiiq?058d`+ge&-j=-4q`Yxr)SQ(Rr$^E*Z<3R7o=!-0=1i?g zT19I*mv+jv)a6|m-gapJ*FA0ChJv^0qsNrg?`gdexLa?wdAkz4t-Pm+cq{Uh zH$UD;5w*)(qB$GhhWy8ecC&fA3cM}6;ObH#_Zue=vR^5W6qjlh!<%nc>^9cgLU~)( z_tv{?-bRAA(bv@AEo7&hScp#DLovLCOu%=oG_gW?8+81S|6=nt3cPiDpayT;sq)t* zb$^Y(Jkn&OIQyYyPEDMiY~G~I%s}ny=UQs!q2^>5r)D1d^YO;a)T}~X_L%pybU|8C3+~L&iY~IF# zw>uxM!5eFBuG-~o^WG_`OSDtN+e3RF>-_&nSA(}%UB;KTyr=I@AmkKM+)E+-a4kv8 z{*?Ld!txv5-n@MBQu}@z2i}HGt;3tpLZ$0=1@3J#U(9^stli zjtp<#{MdkR*}RPhZ=F`x;f?vJSxeHKDm}~-OEW*Ul;7n|vr6)4;*v+|&^IpScX^YX zz7;LM;qA49KKoCbw+Z0w_3jf&Ti#Rkd&*CcHSM-vF*RG<KO@n$S8fnQCbS&KSFE#`E2V|8k-l5~dO@OI7bzSZ64Z6bKP z^Ra}yNj;~CYgnbHate&3=qe;)22B z$QokuvzkQ{>e2Ak{6D zw?+Hi(%a^326%hxxtzRdCeSF4<_2ip`^|T%h4R*M>+#ON*ESQpt+{-1Df2y*J~dL@ zcN($$8phaj#G)TdYp3JchTA4_yCso3~luZN`QgypdaKD}USfNK)K)E&bHYFU2Jf z{nYt*V+Lwg*J!Slw3Nr!pUs;lo5oY_*6{Z5qEo+X^EMm2O&M}+Df2y*b{+DG4q3~7 zLh|I}Emn2rNv*Sai&)+8*5j@JbpHR48^BxtWjT3^*k|2)GDWR$-codxB4?qzy>kBl zb+PZaIpFQ>mm>2Pa_7XFwx)-0-lQs0JoWCRy4k!*t2EtOYt8-klk@glWAk<+cza~{ zb*0Sr)XLd*7fEqVpD8@P{e<)8y9-T+8jq$f^JMd;$tHPNrR*`h4H~`n_cm{H!P}f$ z>hLDjYV~;K6s(E0eVST*ckDkdZy6IayzTX~es9{m%>!>&y;g@eDZdowPJOj+PG3L! zDgJ(A^;lXnfw;V7bZ&UN`OBYJV)Hg1ymcBmrIh)eX5_DV8)zoro72~?>7jXe({w9M zK(n?`-o}pH)%pF?1>kM%?Gbq6&u2Ip-#FL$YSxl8b5>44oiSd+=-EmoCs9>d!~pIGn} zo41AF?WQqPOHtp`i2c`muVdw}oPv~%pL)rYBA%jdHgBv*onv=~w_WyM>HPWnMc{4H zU6FZfl^dgS3a!#@j?-`Zv}W@blG$2ocsroWuKU~f+hXw6xBawI=KU7yM5NCSfw>BkkJdO>zeA!ti!f*F{I#_uEqNw(8ypys=N;y`=U_y~X9t zcNbDujK|V0Z_FIAbHm#)Us!WemA94KZga#QNB;frNxhf%ntn+6{kieN>7~r)l&wjc zSFfa{wME7`+PJtCxeMP~h2pmR(Y)MIrhk_=X_XWwR~irfMe0I%Tl-R{JDvOO*eY*#U0Z{vJz#qd_{+`OmPT)%ophqvRZyiI+iiFlJHmLi^yw~$VJ zSqiDewZ-sOu4s7sXx|6ka(Fww%G=^gI&RFqm-dIu6?1b9#8{Q7wpLwcZ>$?jYfeN{OOvN^3Yu)K z?#bp&nw4+7JSD^1Js&$A|J?4TUoV_kwH+o#&QdCcole>^K>*nK)8CZpJ>axf1_JObeXEExbjafZ^@3cKhRx*Khju!l_l>o||2RH!{rg)x_zMv?izIiIuZ5 zZ7G{3&JHEbJoGTG(VDfAr?ON2S5uK5X3o%tw;@k&^K0ilJ*~>yhNo)qCgsf0L)|yd zH>a;(+Dr6Lcio*zoy3YulObtwcg>1rZFrl#u1^n#x6`Y*H9y7VyxIIxZ!T{d1~eY2 zqVHswN2)8uSt~AYtjG?_+VD2MwEssO-p;7mXyj^n0XDo?Yeb z>E|QxCiN2%PtEX_`z`te3~&3M{rUCI{dP{3x7!EJDHZabO6;ZPN~)`$c8U&}lUnLF zBwpDm{~MAmW3A!sKkxZvM~An6R(YGUxWT+-^zZc@xRggaO+9Z?=2nT5w4}Q>yxq3q z=leOlom=H?;Kt3IH;Ia*ldtNX@^C~pTJHgmee+j&*q-X3yesiE&_$RtD7 zHkP-L&bd~@YN5Q{bnn)mb9g(y%G=}1T7WmrRCx*--B{j2cI3OZP~Ljp+wKi~Biu0LuXDt}=tdFZF+TE>}&njUKApj2(v9x=BBqAI&N>DC&&1=g+m?Xt4}q+g%+7y|1jj z*R53yZ_O|nJ#W4<Af?IQ5@z=|5Y@jGE^=4V`^xfhL}9g=PT=jHw`%al{MUj&P zQs%+%Hetb6@f-G)`>iW@+c3TkZ>`Ri)W3VbNzPV}>v;>=X_-~S+jn-o$KkCTc)S0; z7UM1OGuUKo5^_uCj? zeo4p56PGvNt~fnvY1f()XC7*oH{WihuB^D>ZS2nf{g};L5AZf~QVrg$JYGD+TC16G zNUgZMg>;g7t;-uLMx3+Z?eG=5^|E>E3EqZ0RD-uzJ%r?u)=F{CP@}i-7JUMSxBkbR zd!o%-FYwl(^TLwzo@&mknLQdNqffhOUxCU_`QJomXLvhg`?=PHxfHxDTUmoQ-^uX%Pikobj7zl` z_syxvpKtO~C+wEhV`(YB)DP%GqoZPA8 zXS`6}4jj6@^Yf5C;H}G=8oWt6ZS^=$O3xd2%?yk;vo4gk&R_q^w)Xwj7rZ^+b8)Gm z@2PZxQd~M&jh~);yoFSk)-uix8K>5G=r5GFC9{S7LeE)Xv_|x~cdFv0}Ry

  • Gk#(5du2A{sy?`_@&fw$-SE-g9lsWc@ip3y@? zc+>Q`g}2HCG_z8kui@?4dEfo2&D&t`wqjll-k9IFA74Lf6{+*_rrDjwQ~XXV`^~p& zWw(a68|L-xWAkL;(s#wk%tYo$2X#?mA$?yjv} z7~T&5+djA1ybS?wBL*%jso&FB6OfpZ;<4&7Pi68&R%CVRkRCK^^YO+GsWU1X-aa<{ z&@nb|L&4jH3v2ME%-?oL(H~1^>?=xL{U<%vjvAFGE^isTFuXl{&0az(>i*N+{Jzo5MG=VeBLw+mkN;Eg>bx|4G=$$qG%o?YI! zikcayB@g4&zH22vb5he!oy{9FNSfI)JchTA&VF*JeZP$cZ*O0DOUZdp`RR}}duT=N z^45yG(Da$ z@p2x++pW7E@O}F|y&Ak-Kl;{^^PWoiQ`}c6a!UO(p6}Bwbt}bLF}0R@aCuAAiQ(;) z0Y7=%=4~8!8+@k+Z~QD!vwy}VO~0hMR`M`TKQ%qnl~a(Mj7yr?Qfu}g)uM+wA8*XS ziWzak+vq-zJAeP%c<|Qm9S`1Ex7BsN`z^(;rF$t>t$e)YsBU=s>u(>t!@l1pfVa1< zzOCfEr>Q59V=ryrr^V{6d3e)I!*|#87s}gzzjy2FY~HQ`Z>#R9!JB3Rz8+4%cWnxP zYw;E`fkJt^{JCqhYwk7 z^L8zG>)2^UN&TL-;@Z|4^H96I@v9NF;qAL$=(U~A+jZdW>B%*C z3)ywZTJAko9_FOZ<}Gz~c9L4><1NPo3~zl;{#<99w<+Ll`NK7MBr!Ua-*SZ7O)XuFD-I=RM`F**SZlM-!)qnt7<{VGokW*B{Os zJCQWI(s<|((VSQ~Z|t01S#iVLv~L~1tIgXq@HTR44c?e@bJdko;JIwxJ0*48sb&Hi zPdTUIZRObOUbA_d4&J)1uECqdZQFObQk?xuT3Rc`r983X;rE-=iL@ieeYK{Rl#`@Y zG@Z=MI5jjmnRPaAu{!5TQHSLB&6CYr zNGH}>!&}c?Ke^K8Z5DW&e#u>>sPC!OL)+WOxbMVbt!18kyoGem>P2cjZ-Kirymi~* zD(CNin+@Iu&Gz7p-xW|x(~~?>oOv{!5P#(qScTd*bF8(@BWd;~X_Dje#>{jw&dgGr zp0YN)joIUd_V)dD19-daDG%PP{59{cR`KM!-=y8~)Fe-%;x2EAI(hf|w^#nW%;s$l zczgY_yG!c#G-Rs$j4%)5(j4fQJeoXQ8_rvbuGpbQOS@(~E^o}t4$E=F+vry=UTX7p zBY1n_Mi1VYU(%XWkk-3GE!X1BgL7Ay-9w0xHg8T0kmZQd4ux3{0G!5h162JJfsP0y86Xy*4cWY=7)=Z$k? z2F5eA;cdyjhyL2;?I!TH^74C2>i3j9XVlI8a}sHfQaq%e$|*2|q`6ai7>}h>|! znk{#v@s$0Bw@%OP_eYzzh2U-3A`jl!L#yk`DKzsgl&4(x)1A$mW&+Ai3~!5n@%I;O z-WGwki5psgH|2ioPM))Ee%{z^h|b4bj+qc3guF?8rjA>4sL62U6jFDh z$*<{FlSlHTR8qE&r*Tm_e zmU?D>mp6Wunwh0|hR5*s@RS37=Rtarym3}> zc?;no^;(y=L=_EhAL{=e{PjqizJEKh%G==f50!G=Z#=t1{nz}piKdpOw{YJ0Jv_Ch z!y-K{Z<_f?9df7C<(!7MZ*IKw8RvfcN0qk=$JgO4q&KN&DW?>d@-WW5OPcwhpn7EnqCe2FE8;Rv9P)k*$IM*`2#?Q5qpYeRWF@vUJ z*<*MccFaDXaqhQMtGr$L;1=GF)w9P_2+O5^eE+}H2&mZ&Sk+gl3`Jk+`0POtLz z^tCm3^W9M^`ono^m0i2MCFR46|Et}<*nIvC+$LtvwwPA-uP8&X2_upZ`=2|YJtPs*;U@2 z>9MNRl<#Sd{d&FE`FIOCoshNCG&DK0d6P2dh#TI1_mx|Zb9g(a%G+%-6Y{2+hO{rP zrI!4@@qE0oN{X(SKbCfRi6@)7Q^wOS*Y@lRAtQFO;_#=N#?+e*Jk> z-rk&Dhd0fxQ}Cdv8{%mm-lXYCao=4R!&|vq^Pc{G!k^|l_uKhZ-c~*pfj8d?#G0gU zUHY4cH+Gdm$K@^K>P2U>R~o- z7l5~(>+0~v&aM1yzu?q(tRF_Vt<~g|)-o<>DZl&a_RSz=mg3AWd89bkQfnqp(%fB3 z6@BC7P7H77zxDSKHg6Y#w>SE)DQ(v8sdOUgChvPvsr}i!N%P6kmErBM(jh0>ymbU` zs}^kLyh(FwPTnG(e%lL@&6_lz9C5?jqJ#QeWb<|rcw78@i}A)2q4u3mIB&kyef_>W zlKh48cJl{*Z2fy5UplfAc)NPgVi1PC?4x z8)p@+m9)k$d3@vPc;oKMU77pslyg6_*1q2^25-X_*Wk@}NA&w@_DrqGBl#JZbmbH@ zReU{CMc+L1`{ty-P~Ps^ZusAA-a3P~z8h=s#%{T5YI>;Y@x8~mHnrCCChbnsAt#{m z)8kuL^5^4?*{Ineb$O@eep~wOH+Qjl>jK_74t>1jyr<05D6QFJNN;-HG*wb}tLcFr za-NSjDI4RKHoSdt!H^Sd-nxRfb;}yS8>eCQ-1e&~#(A35^k_8KR!)HtW?-CJ@-R*< zt(83V`)cN-c6noFI;rWAH2r05cw4^k>eFoAx`DUHUv2^3tbNsF($^FBe)Fwfq~GN& zQFX)HV;^6Kzn{GF`&r$=+sz}MC^_#bd5+Z^b4prMi)%ytl~a(iY2v)4skxSMO}82k z^HXyz>*4J%+dU+_ydcfPICx9^h@xtu=UKo>tVFQ?8sstL#p*BWW*s-a@LB zs~g@v|ITHX+wW;l@HXQ02J@!*>~IbmzchKq^}K~tWaq3-t?|$=X{k>=Z&GGu-0*hF zJKMCgdFut>iSkdtI%l&75 z$>yyOcpKN@>5}?>3+$&ZpNZyBvu^Y7mU`DNZ;8&)@OIw9L0j9r^#yO2PN>5h?@wyq zo~5-?+&90kzj=7`?cCR&&0EO)t+j@?fA046BW&LKfw#Bs^We>Qr}SIBwcR<4r_fRl zE^nIZQZ`Lo^02PPz#FZlPU(r4R1f6-hYP8+W_$P?4$_1#hOgue%qLo^03d!DM-!~ zaaL!3DNav_j>{Xnr7m}9cw0Pg^8eYq4FqquJ`{nskZCk(ZRHd=G3r=drO3(NH2Jf6 zlQKwgR=2d_?a7|g{%P}eIe44Y`I(aQp0ZZzSHtSL?T%s(lE*jBy;M$t5o+Je^fQk} zbFHMMJd&UBY~GletElOrrl+h8Z!@RuJICg25O^Cgr3PUnkq_<;cfQVuYA|$Z7_K2x~c|moB%cJN^yELnt7;Gug&I- zooKYCTgfBkm;AnQJ#XwpLoz!l`weeDeEi#gwRyV&yuICRUCDV*nMb32d(-69ctYw{ zPQf>Wp7DKHzqO6^;nero1qU5#cQYH2M!*}VC7ML%`9qT%heAN+KM&D&7$wrWic-lSb@ zPH~sFR@@yw(bU{aHg9wWYQx)$&+hgyo3~-$ZBeghOSnmjKLu$@fqSocXEJB@P8-z~ zJB+0Zh4Qwc-<|_)-mV01FZO<} zSvdtM1D%c1Qs=(?g!88C#PBxn+QH6yItsi^?Du@hc~5L0 zV|e@N4$nA$U-TI8_Uf}9yh;3M`te<>@zWz|_C}AMH?CqfNz+d)`BTST-ZT|`J1_eU zZ;yTXx6AGOZ7g_OH}Hj0=6lM|0=1@RU%w`2ERUYISQQ(UCm(M)x;4C=e9337v3a{1 zyscPRhd0)3Hhx0-X)JFco%2LAI-EC6w`Gsv?Gtaz>}~Tl4!m8rz7B7iTAE!qo2Rk7 z6*-M;-Xfl_;qAH4tbM}fZ9I4zaK-vk=6kA{OvIj}|I9?{-{no(g>PJ{DCLp-vEtdh z#i~-yV|d&95A!}`^ELszy}l#@ZxMU0$y>-tNo!r+q+LiEeB)AnORnhTmGl#rkGC8h8s2{NonPN*^EL^*4ZXR=c*`+| zny1-V-cs|-{P}oe2BkKsg->`X`4Bjq!)q^+btfjcJ&zd}>Sj#@s@y04CcEOyQ zTG_m5GO%KXHoRRjd;a%q-mV32Ph7dN6!krg*nd(E^YNzHo#bI$qccv{Yckoq#i~-y zV|bhT%KW=*-mU|0i*Ad|oAmo2QrvgH%)>twLd~4?Gp^@N%0Ol$4=d8oxTKj=($cP( zQ_q`Jh1uv~ysQmx-`cbHr)}P*fVVksMCQ%v!RwP)V~%;t?B%f z&D&J)HgfcfrQr88gZbRN@ivh1)5BTOFL^4bKqoU$(<5b*;>;sy=9fI0IP*{!!&|wc z;cY>`uMM_&n+D!4y|V^y>{im806mPS(5ywxT2h>z$|*2HEoG46TpLSkYSAMxpUoRH zNS!M^hPNZOnf-g4x9Q-m(>pbIlj?3x@oe6tvuni(Wb+nrM~1gy=e{x9=4}ReTX*$K zrQr88V$U_--xQ~x&0FdnN%_<5l+U>2x0&GW_Iv8^Cf%h{-0Gnw zKOr8euJ3%Zd6TMybe>|ZX4fPwd)I*-8r~i}=x1NFdAlCGU4OyLrNZww&6G5rkohE? zeDqIQ_)bY(NVbf%hPNg2UOd?5Z5DVNdQA=9`01u*trT(Joa~31d8p}ed6S%6MGv*) z(ZuPYW`1gVBuzg_b9XIekmAftPdRRQ+q3^mpSO9N4c-PlP=hz-(P*g$X)XOBn)#WB zaY|@_=bHLlHlN-RB)n^obd~@n~^Ud6B z{z7@%ZR>rVf3NLE@V4^dI=nT*r!_?n&BI%Y-I4i1d0YAZXK%9ax4Gc$rY^6R3V%;I zjab_E)Lhv;>=MR~1xPbVIK+-jS*dEjl-)CTj$?@=|=4w(t_yS(wM zo0Hb_7PxD}+xlIP*u~~;K6vZ7x**=j6}7ZSaz#Hi*D_8md9rz92B{M%&WhB`Nlg#6 z#-s7OylFDiL(PiRdiKmi*bg zah0!TC%$Vne&75qZ<;E;o^n@)x7QyY?fm)ro50&EGdy^ccFIq>Z=aH1sztx#VVs(4 zE2kiRl}>u7>4~LN2^x$G*xo!+T3ri-o4-i z`}?=W;BE5kCgM#qhghC$-f~R4?oX{#{y z=4~l>>)PjyQs#Tgy6mkLHTROuTPyBP>RHblGjLXnXK2IQ*%NmAK$W+Z+ir8jo=5)u z@JYRw_nLl4`A&NN#yYrVq|be&HABuVxJ93U!R`LvzHxmOx2-C?9aZJ+-ecY~tl+-(oe0a#hlr^F&i~AD61RZ zUV7-vJsjSStMaz~^0!J&`JPJq<=#WI%Nw)Bsvav3d-K)Vyh+(|bZ&T?v1%Lq^Yxp) ze>=X)+g*$5@TRHD&xNEld8EBFCq0^+(%Nj^SVg0yDpFj^>ANH5r)K_qyfK5+l@&L< z?KXJ-Y0mw2LY23<8|v`Jx*^)?t>*h1l3!XY)s4%WR9zF7I@GMyc(QrZRM&XQofzJ> z`|%#fJG`A(P5+u&%tI~JWjvcVR@Y>vpPCgl+IOwS@AAe~ z)S3?I;o7n`ygk2a*EJ4r|ETiTds!Xcn4em63iMNREj2y9x*a|>nzfZvh*e!vu}Ba1 z9-97xB5^e_f6hd~|%xm49DyLfWJI|1{si&09Tg2*yx4p;ibdd<^Om}EO@56> z`mU903+3&IJ^vH`+{dP$znxa)t^LUMA9UVRb}OBo#v`q5RGd9mPJt0kSH6Bt&Jd5L zpF(+it^LLGo%`+dDsQWAufbc$1gu{p*L+zd591*{NNbtjz2Ep%O$LphdFa<@uBDdN zN*c=0XU+Y_;SO(SRe2lU{(^k>TgU{OVQuleX?CqVS+DPk!+FzmUiQ5Ep1$^re{*;{ zyUJVF@mqLnvHzc3dHQWkrsK`}bQs>Q`sv~L=NC5pJmj1zZ|m=^!JCz*t#jXvAj zvmVLGTSncs^KE8NI^LRX*LvOpJ2$+2>X8w9S1-8A{dNI(yRz#=`5sj3H#2fGCgow= zcMe=j%{=LNlPa>huSer&e(HR@NfjBVE>|?Xowv*Jui3m^2;K%x%gGyelv?wirw*x` z%^Rywhg59T+I+nEc3Ym3;q7C8e(TRRZymwg8>{Q^*6P!i=BDRO%FxVlo?0=ym8W5N zJMw@_cD8xD2)wQ8(J9}1DovX^l|0Q5&*rVj4r5JB%2OzBpZUuZr`Wu80&h3XjLaKP zN}7PLhkj~Je#x)r&9@3GHiIsPxAK(C{nl&n`txnxE(UL-9?!{}bP7_Olc$H89%_w; zevQ`irpeaKo?>__PvBkN_WI84HgBE5TjxtJj{BZA^W@oEI^I%zI@r0d&c~bdM{zQ$ z8{Uq4@uvN3-nxLdXJ`Kl@kSET@s@ESxqoUDk~1G~IVu|7zA$;||Fe1P3f}H|Dj{z? z-yHoXJ)JGQRVKinp;7yuSNW_BZ>Mf^kM-}L?s;T4@HV?o=hk{pV=>+Jo|+PJ)NhSpBa{dUMDf9!1Y)&sm< zw5|?sykn`YT5WHi#v`pQGF~|aPLG{a$LfkXsbl4r^2Fs$s#1;{-afKsSL@%S-1Equ z;B7y8` z*mLC+7?E`9&ZW9i+_xV+Z-Euf{kHVyKN@25)*HMHTU>)T?kKQ_HddL3`Kf(9^iwk@ zH9eJ6(A*nbC1qorI@Vg>JoM{%({xS`b-AM9ZS1`>zHjq(8F(AGu?}y{8B1$USyL+{ zPvsOeRhro&O@Q%yy!r0BT+#6M?Csy2V)NDqyuC4`oBN*n)@nun=E0k0^2P90?!@qR z=2bJM+Pw7zZ)=vd7;k(jIBmwM>2W{x8mHu8W_qaU3DHtcDb8Aj@^j&QM zcsVj}=i#r*q~-)PQ=&)8!#K6HmY(eUjZSK2mNfku&B;)6Eo;%E(R$ukg_>0ur!ITU z{dVk}V|KTB>kr;mkLbSTJ*iZ zkmB?-N=v(+Abv?8lZ6JT63 zJ&h-uH&&Fiax#XuE))DZgfSQqLiAO*n5A#b}T3a~< z5-2&PxNk)%Pl`Bub9qZqv7FQJ_R}2)Y_#vU!Qk!rw{r5P={e*yQ|H&;Z<QTnt6A{@D?}$!`sW&nTI-?H>oS%c)6nCZTAiPonZ5JC3sss zsRnP&@Tv6OudhEHZ_PG6J#T@X8{T$VGVWZPx2wS0!iQ?`#!rW&rStXmGmo^^cMckV zq=U)3-&|JoGx$tDpzT=w6+|7m$xr`a)QmS~F@!&0DZ@Iq*^PRTTb2x8O=4Ok# zyv3TKwNt}e|4+q=XXccl3x>d?>EgVjmPTH@V5BA?SEkNHWs|yG2MeV zD}NgsQe4w>h$rOqvv~{YO0za(V$xd1IhkzUxJuJ`hR5*s+Km0ruz9;0yv<9>6A&D%Kec2%#-(!HnB1VZAmW|hrbtSTWp zjkPwMH_bH49>d$p6Q(Y;c^ePjI?ZY^-r}A@tZdDk$K@@g+YI&$Z|%k()Wha&0(e{Z zL{8pfz5DFnH)n`H`+j3)YRyS;NAyd17?;*E4>i|H@wmLP3U#@2!`nXBKJ5Jd!ZqOS z{@#7!zNfwuW6!=iq*gX>nodHhYx4Vgq<-@8CS}X$%JBBdgC{!wKlw!PHfv5!-ZY;q z$-|x{j}+IerAN})yh+uiIICzp^lNlT&V0Q2RxJAsZy*1kR}Qk@(@EfM;@UdANp%^g z4w;@NkK|#Tn*A{D^2V=fG&2{erQX;BJuYwjDpyIG{tRt+`|ZI!`rEus25;m0^^Nf>EBb<()2@*q?yy|zwVvJwahQY z>B;7e8K{{-(jk7Xm9$isajwngOhb*X500l`?4J%p>hulSkuWerooh=Z&i*Ep=7)8{Yap-Ol;{k*0&UF<11@ zeowgzsa7+@efxBIYldB<=t0k0V0FXWbql`K$-duafVUn?>+lxRgN6Z(C!_~yZ93i> zwWI8vk~+y57&p9q^T|C&+PuvKZyR1r$Xj6lUT?Q=t;+nHjSb&jX#6BC-KVVFiQ(<1 z^S=Cu&D-_h?Y?0Hvfoq9j_7HX6HGe!q@NT{ThcrQmp6V@%8(;&c>B~xUwDtr+br;Q z!_AR-%h9LT>G;k{^0zK;+!3`jd6zfIsmUg-EyoRS!_J&?zs=ih@HXt#$h-yi?8RQj z-s|S6oPwrXjVGj&)NA#;`F6s|l(pgQEBkib-RA8E@OHsf1LMA@zCAa}U;O>X(3FktVR+l=`dU42sXIxXKb$vZ zhlaO}gWmJF&D#R-_VU$(-1k(OKRN?N)@^>?VtsP6d5cx0e0GMnA78%H-ZpPHfwu?m zX<^<%W~JvXq)L&WAgNY1Z_+B?czH^Ox4%sP#fdg=3&Gpm3kK)A-FH;BCY;IeD{cHPwVQz3F+=Ok4B+WNNxf;mPJrvpZ#1 zhPVIz&^I047K68o9`N8z)0@Vl>^};_Qa^g$q;93mlD|>$Y~EsZRi1|7ZI2_q@@e~i zTLRvmyy%MT_cYd2W95l8pK#t{bs|khiu=}0$D7rm;qB_Zf93ps)>81cY;qmmq}}o7 ze*BlO>6ha4Fc0IBW=?AQH91}0n2nm5Q)}x4+irL%uHPxC)2!l7>8GZrtPO7m-SKbz zs`uNa}(Ay$4pZ-Kiuyq*8P?q@l?9aH6P%bGK$yG>B z&03eYL_fSSyq)#r^1U41j;-=`PxqnO?`g)jhc%}sofPBDN$u;A{ESnxR`z>JCo@pf z!*gexnrlO}lvC=#<&D*)Z1gC#;q9xx|In)rZ^u=6TQ;K(Z^~M^*^8B*)7Iq7=1r3! zg~xXnQV-0N&6|`VN8Ips+e_;(C~v<%`+}z&-u_YLZPk+=ylFn6 z8jm!YBIDV-waP9u6DyRrKdd?g|2xvApZA_rMF8j@UI{1)2k2?3;$yMIQ+~~oZZx79UA}(*6)9yHVmp5srQk>Z`wBc>%U)-{b z!`mrU-nu{I!COYHhJ0TaGM~yRgk(rPjaa?8yrt;8-0i!+KE3bk#~j{Ht@8GKzY)c} zrz!RrlGDB4Lb5gUT0L)pgd5(rYrpr64sWMbdAobQ2X9h8QoQ+3p_x0Bdh?yO%bRZo zsS_)1c-#A$;RidsonGZ_#&aILS@|3ChDts7&Tz~97JUMSw?F=R+P56u+EsZQa`~0n z@2T{YOzGTH#5FyTH~J-AnKr*Fbxsepub&-CYc(Eet<;6lczG-ZWLp6%B94PFi}p!`nGk-p0HXnK$mVnRRRN7Bb0j z-Za(AOd8&%tQn4fj&jp``p+tFJ%^8U-&0Ng)E?=cWSrW!pKRWw%)W8ot|Y%ySMoEi z=Z#gUnT>Jkvd8fD@9V$U(YfEwt@764mdLy*@7}g|mV41?O@581ataz}tge`cb*br* zw8l@5o;RtYGH!S~ZP6`Xb9g(i%G;_}Yw*Tym9^UV30W)EN|7`Bev>+3Td#x57zymbU`Gv2Ag8*g%It5)>;iuqZWnjT;6@}{|Se5<6$ zsp)}Jipv`_QE&^{O$BoH;Pc^%1MUTr{k-I38!??UTyu zc(d*w!`o+jo_nOtTW9cg-!+kWV-ME*Uh@vrczknadEYNXR=ycpif6%B8nJ8&-kf~3mtFLVQM-L9>}o3v}j zeKn_{(af*$OCIKFPR69pL*j+WT(x%wiw<>c%!Bf&YN-< z=6-wOL%sL6dFuh*ZtpVAeNXv4A$5w0F+cakxNlu)E#oe4{3^9?b^4iyyQ7CXmWOr2 zd1HsvR=0+?a~A*XDx0^S;O&N~HF(R&>Ggd(^H5uTM#;(i8tw9?Sry{Z>{{dT?IA92 z866tlKD^(+-`KqM0&l}s*Witx4(?8h`+8#eefy!`<&9sZX1CPzNLu5kC*w5hek!FN zT;3!nJE2DzH@rRezK=RT@4W=P4el}CeNUO66OekK-&ZrIREr*J-&*v$ylJX419hyf zG@YhtV=DerKfTVj7Y1bIM-53eksniA)hAZVXbW5 zxQdz?l-lq%YX5!l4;oc|{&pF7d-{?I+3%^cR@-+MoEUXT9^bX}OFg*vo8)A+A~pM= z_U$2?H{Wa-e#6_|i+}br`#tRg-j>de%v(l3ZA=#Fkxt6xP3lU@K)>XX;#`|r7s}fo zPuj`(d2e6vHsPs)c#AdpSa~X^;G50YpMoXd9$elu8R((T*pa#4Ca(Xe^Z%3g18?2? zT$ArTmEL;BsoAF#r$?ii$5-p`H%%w>NS)9lX=$w#m+~;~^2V=fG^$T3lr&dS)34bb{ZbyrHJUjk?eZo$ z8JBiMkJX{!?bz;DEwOnU0N&nsrUq|To;C&;XFrmrU(#F~;*r)$asB)UHN++_V1}9u7Z|9u;Vf=$1 zmEW%)2HwUk^Wcp?S>lc~6QCz#E%Ru!RBQ9&EqkY=PBS-pq|Ee`wYlG(yx_}&?E7sv zcYeubI*05*()^26OpIq7kE(^RBKQ>9$d z+;1m-@jB<{y`#X}B`Z95V-MU>kvdin`FJaGr?G0eyv53F?^-s7!TgM-dBS+*4@E9#n*iR{UpOt_d+OV#pyVE{4Zxg}W zya%`Nw&~|@JTGcaRf^NoY&surId*Dz`;Xp#yWi$*5_r4n;^~cjPo>kzv0ty-qnP_m z+Fg7}scN zE#t8?`*C^WS6Pv9jh5C*apt7QmkQ>gc6nnJI;rVVYQx);qdJYUd7A>>ZlC7Co3gIg z_fd_?>GG!O#Fqh0&eR^4w?w-zyv@A*N3YwwO$BdLAKk)RLH~bXU1;?>9QBrOb>=eksni8m-wW z^SHeEW>9u+cx%7m&2Vo%-&JYg1_Ulg(R->P6;scS`CKbz*p1G3pP~ zY~E&ow`U(u$XlX))x1|(i_@m2C!05E6*GkB)cM&PNz2|fSLM)#w_acT%ZoN|Gr`;K zmtL=bPdO!O-`+yj`qq;CNhja-iD;gkk~+>LVuyyeuRJ*Qzii&F2XEKj;K3XFq>k0I zZysMid#=nsBtytrcH*mj>yor|pR#gShPNL~J9U=L+br;Q+0!1pDZg)Q`*izGlKH89 z^U%-JVa{}uXLVoA4t>{3{$hA5S2w(U`p}&g*}TmLZ}0S(rGHPQ9ckkHyGJyBddQ3v zmpt@SHxF;VopTo&?eeCnsPS0U4R5zRefjA&Z#RIqN9Gp9o7G$HJ1|zB;(5z)f>EaA z@)m1`RvyFK+E<4C#O7@dcw4xx#dwQ#YI@#cRrJk6Kectrb(t)Nw{nMux2uoX@nD;` z8^PP?0khrrRQfK0aY_45!`Cm>Wt^wOIBV&7<0@)qrlyCQ9%_0dO+U5dXFQxYR-|T? zvNpU89r*goHg9vm+mM@T@Mh(y`3d5g($9YAVO;VvuF=d%Ev==;<&9sJGBB?3(<5o- z)J%+ijn2oLrdw7o`wegHe*48YZQkaAx85)0DU8{GFaN3EK7pe9d|jIp2Q;Vtz93gzvpyQ_Gixe7vy}Wkti=X*+d%(&lXucpJQY3vUJe|CTvUuv5BQ+2`9i^HBTdp`UrE z=_wSqS6*CwLKU~ID!d(41^RG`UDJb|GC?DA8~j)rpnt>Z{_5Tw>NbP#x(hTJ?^xn3{sps zrANBe>ET*xX)Qgz+P&X2ov@11Z+Khr&YT?_-j1#EcHh`}`QFnMli?mE9jmU(n^dvY z<1TNpc4VD};qCa5A9>Z`?YJs$v+vHyTdbZ_k|-+U+5Z2ryTe#qSkIep zX3KAQ8#ey%c@A&?sPcAu#|5qRo{|b`YwvBRFL~0vr&2|!!_@IYdAswg!$0ltc2bqM z>n26ut=T55Il*+iX{yKaXeM7MZ#!>&^H_(sldHT9e8_`0&F(l!Uq9ET*5SPQ?v8$H zX{Q>Gw3cy~w^*62Jmx)p;pstx9o|l<^7iV*H?`J#%1MT3mp5yAZ7*_2m1bD$@|LJ8 z!&|$qlm6-Oc50QkmDlCuP5CqDny1E-mGaONG9Tt)EtfY|p;OY5Uy3siJ*jnE-Z)Qo zUXB~yroH{fAr5b+Re4*svIcKbPAh*+mgtG4HU0S3%I3{CTP%ObPNlUjZ*)=z?!xeP z>lL4#?(lYcmA9+AEp*@0h&?1_Qq!m8shooJ)xgQu+;x$)*nc)}bOvg}+Yu*UvDD$M zU6r?P(`)d?x|&xY)+9BxG#>UZY4?89WRA7dSb4ZR_Rst-Z(J2o8{T%Gc;`V5Z)a3_ z+xTc5-hBJvuBm9u`+ zM2QxBIU5;7zI(D{g&WYJL~wTT99re!oebNEIc2>bRaaO(z+c_Jyzxz^mZoj> z(8i%=t(22-=BK7dqg~#7JEWglQ$57vyX$P;q-@fT7^g1p$nZ9Fmv7*YpKjvq+$wL+ zKV646sUJ>E@>u<}vB$O4QXbzpb4uFfO>%1D(oUtglvCr^c$hz%H#(_VMWf4W4R7BZ zzT$JvdwO1#w+H(!(Z8qCxwFq^(3(Eu^46$b>vu}(*sYX7igT@{4R0s?bcbym-p;S` zcEdam-k3i}Ke_L>SasR6%NxJSiYc_F7CqGLM~c%!(vt4l@HSw`y*j2)~8L+f@9#rmy zI^s;)F4^YfP1>~-*GwjxH)f{RWVSqpx5@YJ{F=?%1>kMvO?7w+?4j*^t=<|QZ%*D) zOeveUh!qWQx4u69DVw(o!P~+Y>hKn^pSp}mzYvPm1M^o-fe}_=oSGg<`}(Ckj58-S zJ(6~LV+Lv|gT_-{Yk2$Bd&f<-dFu$?W)5Dae@~@;7}xv}HTq*|-+MUS{l-ot9c$N` zJT7mVRXIF{xA$y!)JmJTi@@8UC6Rf{(P!OzFGbFDyrt+iBxj+#efn$P`;5(7C-8RR zi#2%T{V8cqz}LgI8qNH%{FPIXGBeIQDpqyoXU>qgv^F1a%%;)hU6}jrhTrV5!RGB^ z@b=cw<#FFrPA`S_J$x8Ia=iq|o`M9gBAE znmXN1dCRe)W=E38iW}bAjlCRy;JNbiw=Upq`6~tSmSU12IrH%rQn68MNm|lf8{YPt zH2W0$e(MU}uD$YR{d-EDsT(y(*7fbDGI`%stQewG)Rl7P+z57_yfh*zyYw~>-dKxwGBrJo(wctUPdZmoH)1{tBpw3a<@V4!ffBu7g zzjX(19pA{wn{Pj{{F=`iJ=E;M<&9r$HZAQ!(`PnsnhZ*h;qCl>r#ru1=mFlI8g+~Q zJyq6i`@E#^hfJRRv=(pd+*jx0EytV@h{=F3O+1aG(A(PF&$PFwOf4{te6 z*6S0k=Pj@k!&}GSeb9MNdx5tpZztqU`i?@1OKflszH8~H&c_=wP_uK!sp-*ZuBG-} zo6Q?DQ~Opa`wed&_~7vZCSb(|>&UBQ|fBfwyNS)Zk5; z4C{tyO@7HE#eMrp$D3~_sr@c*QpLd88s5JD^svv{y!8QZx7=TYH>sbHI6qU=+#}=E zzICOwjJv$?t5QXF8{*+w=A@=a(wd(2yh&9UXGQ9=$M81p)?XZN^VS!JshL@ODA}yC1iC>j&OOOpeT(^a+*Xn!SgtmGX!4#;&NPiXrik{3I=W z*BQGoynT4!8_w_7_XlqsACAmhMn5%w9nmO%(#bdBPDveS?n`op-|%+e!AI?F-){rJ z+tZy_PDY-idx~krRa(z#L^@!eAgLW8Q!|id!@6@+d%Mk>y#S2S>Fe?@xwTE zMh|UkLr#aaG@AXeF4tBj&j>X$XyWupTFM#XXHH4Gys;uzNpX#*yw>nG`?9I4Y~C&h zZ#S>1!5j03Xr6U5DR*cmxa~7%De>QI+U9n0zkKt|7E@!@A^EMc~^_Y>9w~#(HYpuPs zy~~=l^hkNKd1HnUEmaT6>AN-^Z&oLUxBYhM?EL>BSAe(mYwGZ3<;;EVA$d}qS~0wZ z?3!zRCqRFpy#2iW=bS%ZKLotp)9cRc_msPirKRsnr8w(yl8pQ6e7yNq*X&5sxh4-i z)Y-hzDRn{*wdFVW+Yeq_)YpDbhl018W<}u5_Y|x?bKjXEb@TBStMelBWb+o%i8X)2 z+qz>vS;cLu%Ac1R2HwU#>A{3EB^(~Oh#nwXw9iC9g%ylcZ-kKX$~RK4IT_uB~Y_Trp` zym4aGQvVu{w3cz^p{6GtZ>-V`IvsD;Yz=SweQNq*o3|^$+uds;^Jewn^+_!9befYl zPLDcdGU<4;x-z_V?6duPo42dL+x7kK4u4NIlhJti5=okSq=#{8$&-#Z?vz^UgmG%_ zLZhX%E^m@kifgKuJ%+ba5BT#yo41kRZOr_fym6Ase!T9w)cN(irS2rf^qAA-jbDwZ z4R6aI{?PAj-bR799?#|EEu_z8I6*ycAvnufZGhNLuPa6A$r7{YY`Cmiy_Z zlUmBmIQ`TitTK06mR@mhzW0>1G@AWOTFR5c zpMAec6&Vjn(9iA;qATqpXKm24!q4=nv*wW4{g6D4Vjp4E&8Rp?)}D9 z)KUg|QtOZ&is7x?q2cYnRt(SWx22GrvvNpU;`N?TLZQdq=w`X6C%o}@P|0y(UQEO_syh$0z zfRveQsp*&Ez8?CinV;I_jjO1+N)xB2tPO8RpMBS#Y~Ch;w@0tKKl?qka^}9rnk`S| z6q>E86#eLV3#@K04BnRB?!lXee|o3`_mZ1)_CVb{ys@hkQ!13V#XEG~ z&F1Y|@HXSkI=o5Kkm4yO#+*`JDbBUjjpdEqQnN!z)34Em^0w-@uRDMK^g8f%^_T~? zyr<#pX}-sk_(>fvl((ne{C!vZewzZ`F1af*Z;hI~Z{2XlQgae3zssBNtdyOY@84$6 z82>|?x2fRm#djj}rtGcl`>{lOZ_6+BQ#pl3RhPO-5qEh@QKg*I@b*s658q|;HVwQz zFz&%(-qU6~+2ZfFMopkl-bVk=p$lx@rh~T|?=6V8M4zd;_n36b#qcIoFLD~;yeW5K z?zg{fzm@avFU$aMS6=u~`2EITC21z9@oPMqTGFhF;Vq;?&038oA8)?%FZ&H|4}5N+ z^Y7Qs1aCbjM&M04Q_VCQ<*A%Pk)0P=i&M?V8#5@i;qB-V8{TKXr`LnG4G%=%P1#S~ zw~wDGo(|)&YNg|iowHjhP7k%lubE6f-Za(OZCM-MMxU|Wl{Rm)z}r2Y9u9v`S=YBu zUq5Sw=*lUCWYBc3@r3lCS(}eHDRah-3~$q}Jo3FZZ?nPMjB9J~mXWjW9VPXtiN~s| z$&-&aO=j-Y*Uz=STB;S7w~Vd~Z#_o+@HU&b8^GJZM{4jU<&<{h8~4pG`9tFQcw@y_ zyJpT5x=`M}dBcwXZSytVQwDV|Z$yr&yq zdcSkO%>{2uS9|c5QLFCvbyEL~Q-{>zS}AAc6r`^*Ln~_TE1Ng&j;k0~YQx({|KI8# z*!SB!@HVE$%ErE@fqSgWxiqWn`%PMvI-X)0dfozeXLvj1ofSveyv+x1J!eMdEyct* zhgkU=%bTOy%t@~A22*t{(OZyO$~!&}IA4x9}0P}4(A54Ep{e(K68Ff%os zj8oGiX^mfEOtV(1%Xm88tj^8-_V$HmebDCZCh+$7C9AUEQ|6@Rj#^QtyWc{l6tY%2 zg>2p;Ry4d_^!w2l*}N?TZ;NMp@Fwj@ic6nR#;KV{6W7d2%A>#E*p;N&iBwUumY!HS zQSOng#eX0&`oPeY`8OhUZ@pSiFbMBPXX*%SYQfI6+yq)}+ zGdkG!+hXw6ug~i2_cSAa?q3@!`_G+U({t}x`p!gvGY&)lFi#v@V4@qI=pFi#N8FCGybl{x_6_V zw<32W)$%;iepXVY|>wBOEtPm9od+_V|~7y|9YgRu$fks`55^L1f-yok*-a(ya3F zCS?nWbEniHIk~n_-j;voi*Gr+9bM(E>+>~uOR;NCpi!Fp%I1yEMs>@&A^jA}+wT8$ zDgJos={`8%$bMCietGwN@sKLBRljM&QeNRX7 zvsN}^bV}KLiGQ2W-RKi&Oi{c_EG|2E*PLBMU({dQcHw`(`l;4RipxoeG1QI|QX zvw35;T*Y{+=ybeU6EM7$j=1Gu09X0-!tqt!`V4tY|DLw`#_YE7*W4{Pap3#nLMYj~S*Tlc>@_uD_Jye%F6xc)uW^uwKM zG;>ORdSdB(y!lr5y)jZuK+ju9_3~Q7+b4hi2>vcspsUbMepGuF}$%NwQuJkewQ~*wjA9W-p;vi#c+qW)2h5Jd((qAO`pCV zevitvQk))Yjh`MpZ(QYDk*6l*VNQ)^9!WEg%bR8uGte)^%O1nq%qQOaV~4lXtGvw} z^JMmW%89X7t814x%}zBQPDbO`c$m{yySy<&tm@@FhPOwbIJTR^Te~W6!|tlXTZ-P8 zvr*dRtx=t%uA9wU#Oj8(_x^m5`~M@IQRVI8_D^NsZxQ=x`+h#wXDXz(e`Veh&EN30 z?ydzdJNMg}Ro>Q(D~Pv{J%+56nEY4fO`1ty-0=3y*WZhO{&dsN-_EM?cE`O1@fNt3 zw!1M_o(gX%NR~P*vKD(N^gW$<<&8T!_uJW3-lkvpwEjJnex1VIwTjl?Z`?Jt?@qav z+Bc7{KbtppLd`0SQ(GQ$zwNWbH-|X9om1s)?8FGXHR_Yg{rYM>Z@!tE&7aMiZ@1)) zx}3-GcJ_qb&UbkGXO*|h9*n>nd$xLr@(!lR>GCF3(R`Y?mYS0Z(Na#vsaebAjZSK= z(r9a~;cfC^H@LrEIJe4Mr;FF>-&6J=oq`nS+D2*i=JHl#SL`HpU6;2+6%B8HpYvJw z_ZQBq^0xNcoVU6}iA;^ph#0B@__{qyzh@%K4Q>iSH+2bE(gQpXyWd%=Zd z4q2<2p7lqeyuK}Vfn&vpog3h`zkAu6Rp2VTbpUUdPOHJ2)Podf|E;2nx!^c4>J(Fw z@@MnLPNWRP2b;GGz}tq^HF(R&>GjEC9`1#jo)E3cNsoKKX{s}W#!rv$ z+7Lh2>Um?gIke&J{YO2HKVV@oM-KIiUi{d>E6A9)dY8~1n(-k3j@=DuR(49Vm2 z7Ltu?Q}5I_zsp;qZVhjTKKqTG?fb10c5Kkk^VS)>J^xf4-mIK$-}8sW*}wEj zXPkc4rH66)sp)Zf<5wlkY}E9J#HF>&&s|G?#`V0h3NvWp<*p2G)3<&8r#5e0z}usJ zp3}dl+$npY)_CY|l+MRnqq+^L>++__=6jBtcg13cpJ$Tdf*(#pO zDfmt`_3lFYk=DArNzTCCnfvWG7v1ap{z5nKHfG(wG;e_uj`H0_>K+(3-F^Qr zrrGalcktG0!1Ma|G*O>PPfhwPS586Nk+jM;F6B=VmwGFdw^!Hg+Qa6p2Y6e*payT! z_lnY#q@7BB##xJ+9!ZDP&E`$2D8<vR3#W=NQr}Ri2GN+`OhnhJVcX{*8%xu)mQ?71! z+xxQ{@D~nNe*d%=c)MlL3*qmnv{UxhY&x4a-)>_~z&DS^@AAgoX)-ANhPSa_82VNF ze!B#`U9&g>Z|sMf{4-8X4>diK*7zk)KHj8k?2sO6Ul0A%Qcj7SY~GkzqowNQwT8EA zuG#uKHgA`Lx8WNj@Fwk4(?312@-U~Q^YJESYvy=d-ZFM-cpI~0XZ#PomG`tac)NJ$ zdiOo$xl?DFr;>;LNNclsV`ge*rk1Ldz}p=! z*Ws-hCML}xq&Js0sYTi zQF@UMN)x2_CMXtch!s@q*ef9T+mr3H_ve|}YxbIV=6&}*C;az)*7?p_Yu0@B%%Pqb z?eZpd!g#1Ib!d1SF{V>*o45Ypt=pD5yv6Lanb($2p_w~O)tk#(s?1g%!`t06=I&?n zHUPZ6H0DM9duruR^8Gw>YF-EWsYBQ5c?-=P(-qf-YN=0`H^~`U(eU=iUw!^ao40}B z?Y3KM@W#5SdSlKQovI%_Z<=mXb)~5l>T!8vhJ>9P-e%o<+<2R}LEvr1wmQ5?y=mTe zuGMJfq+g21c*>{1%#vmm>QKM5mT_uLehn>t-dIu6?8M_Sy!HRp?%%X|8w}niPI$?E zPo9fpiXr< zjpa?+kra>FwaZ&*6{X+sw&{>pj z7MelnH@waNbyxfVhVq|h4Fhj?P7L6!nWw6pN#(xv^A?(o{zgrSb*T&GZPfO6ecis_ zhJ&{`>+0~P`CgIJmNfHFOa8p$Y zQp|fQ^~t^SQz`kSIP+*c+3q(@W{rnCl{%O5NNbsgnxwg(GFG8x25Ne&P7H6)4Vk&X z=4}Lc>o>I^-qz+zfe0g`YP%sWa@t@OH-&|6FSGHVVAmcj>Fy-czYJDbD^S598Ea zE9p?bpSRF#QddRBrJnt~rRuO$-SBqlrky6+yp0BLbEntgjk};u)u$$pR9A{Kr{wYT z#tbc87sFd=N`|+!*I#;`&D$98He`JOZ!wdX&OIh)%-S8iC7*!d?c|@$KG^1MEO={w z*=yO})6lb)c;l~ksHOb$NLpIU9@0)*>O_i59ZGR2j}&)z%3F?Am_dq59?Nfd`{=H( z|Iy}c9C&+mWZ@<`ok;*v+IK4k|QMXL_Uz@k9!CU)B>+sf^CeKrId25E9YI>lDn!BUUk2h8+X>-4A{{ENuw|Scg z-X7__#eGjTyRd3C)eRv1dW!ohp8_MA9Z8uP56wwGHTz+H#x**gH>oSDy5Viv+jly~ z=4}#qyM93p-mE-zSyJ{>H&1BY7{7bJ#bgUz8#=L2Kl_R2joCcf@V4hOA9wz|elmEQ z^h5&QJUuiOW9%~M=UBhqSzTsvF);I{cy8_Wd>my!F0ftNWfx(}=m%rJSL0 z&whj0&c<74w~g|1N@;oX?8xwT#CKNy)aGp}czbPe0B=$)?oRSZajvD7{Hfycyv2O_ zG&_~@Yu0i~E^o|C%~ct+;qB+ApSRxT?Hcg5ep3S8q~16gYRRvOhvt#|@w{oe&D$gG zj&)t$_|+I~RX4nycKXX*Y~H4Ux5Wcq*T1JRy*1m~*1?-*s@Zt6Skip2VtCtl^v7?v zd7BR2#x38$+wm@Y8K$i{k(!>oYh~k2L$c*Dyv_gOGw-o^yB554e=ad^c~8&UW6cxP zcwELbtMc&VJ+b_FvpP4t9sa}4ui3oK0B_F^-j?k>wQAKoca10J1eu>&le2kvle!Jv zk>tSp6DL+6IKPLDt3{QqyWz}vMiHW6owei3zOKkoe&nt^_HLQPMRnmzb=qmw#Ax8{EP)py@|fz8_-@YZQ{ zLA+`Dk6|)&t;-uTP)nU_Jkr`C<9^-}cJgoDhX3RRo42{(?eSNdh_{4$uYBht<#c(I zoGm?GC~yD!r)!*_w>uBKtr_`-{ymMnZ|gF^lVvSQ(@!nkJdEeZ8>_H$b|`7d&p7il z599Qw<&BwHy%ab1+m+w_q{G{M@HYSE0N$8CRI^X&&^+`@d89bk`g!BoaTT@3L%&9+ z%E<}1yz#4=u9P0b+nkkWo@T$N3&7jhEs1&KUZ^!a(@(8gOApsd+Rq!aX*8>7{6%_P z-eRhIc4~M#sO#UHe=oBTy!9X3c31p9Py8wH6G|Pk$2`{RciPOm(Das;H_urZ-j4s= zneFZSZ4r3ea%*DV*pFr!^i#*IWgbcEd6P1#gSQRa0(i@tfmA1` z>9ZK#q)w!GstGVpp}dV=(f2r;wo^0ZvuLMPzoE#Y|?-u|%1 z0V{0YR)DvS>uT_pu!mf^is>^y-eS56Wl7`r^QOt5^cddSJ+b#{o44!0+p3No^zW&% zZiO-4_FOB)>EWqS(?hNC(4URB(9T&=>QIxDc|!dzZ&D|$Lap>0-ah^3kW0R-ml(~ul9uvk<4wvI8fUj$E5$=S^t-(AtD1`Rd-T8Wx9>dX{QJ{Y;O(;e zYVgKSkfd3c9>z5~bS?eVtR=g#?Pl;Q@+ulo` zI<(5$`dxND`YUbzdDPTCt9#Bmtn_`>3)35b8{2G@ZXLL3rdJGZrA`cPJN;&MhbnG6 zm3cd+%G=%R6Y!?_kZC4Uq-O_j$tPfV+wXwhogChdt@5_$vQEW3r~Ed7nv5|{O;0La zo_5|dh4mpAT+np35AdE+Y1gj(`2 zv*j_oeQ?X>M;+cytn#*XZVld8SJFk^MN$tj@s`V5=&oJfLNh4+hPNXR-h_XC`sRCj zQkA!d9?i%bd(g0FeHQB8lguyan4Egvq$--Y)A>v_v#0@7(p@t7X`yv1a;x;4B_`PNn29Ntc;@-}^8 z3+7GwY;uOyOe*)Po;N82t5CBmYK=$ob8V>h^Typ#vlB_PlhRtl+r7`7cbLQ5zpA_q zf3gN|Qhvr+i(2wP>(@E{J!M^~x0pEj)Z~%;jQe@hREgO&*HW`XYN;QM$K_4Rrirs# z%Wrr)e&5r7@7!;vRe8H)X#(D?x|KIeGYzSKDK6Ctjr)0H^%x!6RVqIxpO!afX1Ard z;ces0KTU9WJH5)=3s2|B8~aJ6rTvEH_w$yjN@!Q~r|KaaZyBl^-u7K{EdKkDH{a7U zs=VDfuuJ@V$|*@&)4#?egRu58M-#qpN+TBDkZ<+?L*HT`!45xJG08$ z>=gy^mZAU3yPRq*d2TFk+!4ECT%)D6j2Ft=*u_1kJG`A$LZh zMeqKq^PZktm4R2@k{l*U+-Y%%}*6+1Cys`7v zq_gps@HVLV2~9O&_UZDLs*07z@OIjvfA8t=c43vb*GF~B?>*(N*_-n75_BR`PR7|o z`4sq7YC0uNKegn~JI)@c{k-vvxQcOQ=Z3cf=YIr0e&NmE54oty+taty;4MR~3LEU1 zeKJn1sYO4vW-WW5KimB#Wn*S~iqxD8>-u@)Drpx|oK;HN@b>*dGk)#dZx>g2yJKq& z-mE+omO|soK7;b7${EjF=rkmMsvSxBUEXr4Xn6a^^*3Dv-ZuRE-`BUr|K~7!T=%pO zYEGL8f>Ihi<1TQiI!~2{-NtAaxR`97wFbB!9sX(S=huIwO*`;5IM=$o@vEt{rjxvD>3L(f)XEMGZ;O7o`j^!Uu6)0>2XEcpNX(nEx8(O#Xx(CX z3+*bEpLeXDx5(;-w~zKX`yiXQ4&d#Dt9rQaDZ7pAAvsr4Z&F<9zjgD*T^GvR!ADGa z*5<7vcw4)+25;;(4;{)z9zB##A*Pc?tqq+mAsZ zx6a`0;ZD7p^F3AWu`d4-d&To6b&_hQQl3J2+xF8hI=}zc1-xBo3d8jdqy*vB0a_5Z<^iZG3|KXq|U8w z4R4=%$DP;Myj=#~dUxsVe^0HP6|ST>drReEPD!&Cwd60JH>oSesin?C<7s)5IxNKv zZ@n(~(JyS?x`DT+ug%DtX4g`0nzb<=OZ;CTr2tGd1Gd3RxfG8 z+Xp}O-#@Z>>jB!7DKJ83sAeAO&^$4I)}{9I zMyE!zL;6c`!`n~h-@A*=TTk$|WOe{=tfkRbKNSw8mqLn5wdiLpNi&cCeq&~?VrIsv z>7k}a%1;kwixZ_EWjvq?|l)YRMlHcX^Yt<=u%?H&wh) z-kxdm#l3Cb`hd4r=hfkjH#0S-!4r`@^oQ!uob*$(uAVopqGo1|=30%8$;q`YZ~Us% z2|d*GGhWi>ep~zY3r@Cq>kHnVf2qx29eL@K-?Did z0^YViAHW;W%c@oLd-51h-nEM7EoK@qYxTTIGne94SBAHr-*VJQo429h?V+Llo98`M z?kk9GDQ7m`q->#a>!gBqB(0Tt(DNo`R>lo)XFj>>{WfpIz}vFx6YwVORM}^))0TSB z^VSTzlXjtr7s}f{qx(K(^EMp3jeog`c;m_P&w@2ml|0NzkDfQKYBnwHj(zHR(_~|2 z>e5-lPoHw3Bz9DnCin?>a+u!`s)s|3A*}hl~Pm*Nq-n%zG-` z$(neKN19vO$+JU^W>*?NnWvw5BpuHiD^gn(4R5>k`t}L-_iv-Y+k!O(@n-c_*loa* zVh?F~ld7;{qjWrP%Fh4I+wTuM#pZ1ccpLS4E9Xs`Ra)MpD$Nj&=S}%(GQ3?f^2xny z-o}ErF5?Fk^PY0@%Kme`16wX{q0@`!jc4c4hPOYxeep>)Z{xt*Gq*Q@H&4&GGRC@k z-a^lpRhmH;%G*c(-*2n9?Nt8v3**7top0>mt%d!4q)`5g;jPq_x!-;;<*mc3z?FHM z0N&J6qqM1Z=pMq{OnNjgvLo) zuDdq8ZMf*5O*U^6!P{e#bL36>y+OvQrAhJ>G=3=$*On*G2(>1Ia|z}p@7PJ1Y4NZ|%lS8*1}5 z1-y-(7Qma-19wNQ@oPNHN$uHtg*PdG`4ptDvPvq=TI^iXF?E^K&s(Z0r96hW1!whj zeqP&D@Yd~t0NymcajKHmm*NjusqtTkRl1GY@G0CInP0GMHwX|z`sOe{%nx0VY=Z#gUr7HAT z+VJ+doj>pV{H$r1bDWH6`qc9lnpvuzDvy+3iu-w!R%M7A-e!Dk@sI8A z-=>4Ndmc={8#&L=XXSgX(6uoqsOK%Tt0MiH$!Ft@nW;UyGrYAu|Ij^b-mV32E4mMN z-%}}P%=t<=8Rx#J=}Dzq2XC5bWaCYf+430PetGt{PqlfQ0p2Fh3E<7jU-w=jpVXmw zV*FCw9lWKTKxxi~xBI58y~^foCV1=pNRGV4aFX}hwA0SJ+nD@?-f#QAr|0Kv-e!Tf z*Lq!9%zG-`Gcl8lS-XR`_z75NYVNns47~fjHgB`R+x_!v@Mh(y`)*3-&OFqTzsR^d z?a&TG{YCDMwX)rB8FpuQ`{UeGHrTw)0dI>QufZF;ZLQknt#$8|)M<8_k++Vojyccf zZ7z5ld-(|WJ>{hk32lr&dK9x2YXlIB{Chkj~l zt;-u%Nt)T{k>Vw03~w_oI-sM?+kEo2sRnP%sbP z?hJ1?F6;07|Gf*qTiXF6-S;$AKQTO{>VbLKPw~95I=c$hnmqIr%G>@YwSBvNzbyoB z4=oGejl0lHDb%mYnaWcc&D&6ABczM<5f>gIks?=#Ok|6k1_@OHzq0lZ1Inwcx9 z=jP!p@9Ft@OIY3T_Rqd2{?fkR7K69RgGZ%(Px)Dpw8VfW&Ybi!PAz%-yfFhcv*n># zOVX@Mj}&)#<0@)q$e;~x2XubB^XCgoz}wiBb$B~#z?qd_OJ?XH^V)=;yP7@tc`I_K zdDP{;?KuJKDA#@+jk zcS0T&nX^b;3~!|!8Q%7I?$4ub-j;*6tv4j*t;k7AglKxu^QOrrd15+@S<8O(ym1vZ zJE5khqz!LhJMx2FY~EIYxBFkokvC6oHNU4~|1oQsN762Dk~392j~!`p#`C6}fZ?s% z_78r~=IuK0wq)d(`1h3Wlt#0E$xn|)bMl&;E^k~VRb&;(lPWItBgLhu|YCoON0Csp^m*_g@bd5fu1T5EV4 zcGB;jtn#*gmz|IPYMXx^HMP&`p0f@s?Q+AdO@v#jL(}9*i<{=4G+k-(gnA0at#_LR z_~V70%Df#@|`d5h^bX06LxPUM^G?b!Vu#$SJXlec55yiJ@iuK4Ga z`=yqqs_}%b&5yU3Zlzts#9iK)En&BYxAA-J+OPVYzRBBhRo;5s*Z77k|G=dz9iC_Fnl0Tw1$>xAFv}`ABEsi5uR& z(0jvo9o|l;^0sbb3*b%Kqckfi&OGtc)>L6d`l;!WH2u_)pYd$>8#5@i;qB{dPI$@T z?Zhf?v({zgEp*q)p6lLAF?rat|9+#B6{R>m&8D;QmSLxcw=;H|hyM=$&F|k%s`A#q z;{^YEnxSrmnP!_j`|+h|>o0MOQYkH$6mDcnl)$;Sk zRjfiyPcv(mH}1k>rt&s1PCvD^Ys1^jKYt8={q4>7^j}rpZoc#?|9fiXtn|~UIOibs zpz-K=ld80oxXW8kog3ah_50yJaqhQMtGrF0UV}F>%(G)$>RIwgaVbwsTvJQWo0N@l zYN>O2B+a$-Fs{*D>*tM5DKkA14<(P`?X4Xb|I^{^v?^~C*4N-os>?X*GTu_Op0~Vr zl=me4yrtS*X%~jKuBTjfxx?G(Ro=Q^c6G~oPgBjOb@N7s<9TB?&-@KBcabxcl@=HGQp#Zrfcw@-}u+W&KS zJEzLq;74olruih&L(QHwI&>}l)S6oK=qJxrQboo!nmI!?^Gj>#$;O)|v*j_oeR=1z zhdI2RTji~HpGooWsZ}@Vo@vzDY`n#E%R7oXCZ}d?THZVr4R4SAX8UG`xAUsJy}FDTDAyh#;HyED8U{MMnjIJ})- zj4+cX^XCb4T=ObZM>Ot>*_0 ze8S=Ff+}z8ubAwAPg#roP)i;u&b6UBRZcx`p&im6+O5W~@vsM%H-42`liA}jyzOy+ zmst*P7gl*&v!o7hyc5`)Mr+o3`mcN+sPV8LO=3U57X3)by$GFlUU8spaw((@8U|m8RkHrs-D7CdErR z4R4R!{n73YZx>g28!~W;|2@_8mbXXw`KsK9ntmF~TOK=-cJ1d);)ii+sY)qsc?YE6FnshMA+^}MkPJ7ip{NRPCZ9_G=+B@b)4ym1vZt4mt) zm*R%EudY4y*y;sWzTet|w+%z4HvfYv?L8*0=Pjm6Gpu!a%c(2F+vxqzz&CJZ-a3G{ zm8&xHCUMTa&_hiRH9gdlhw)JD^2UnPyep_#k$!4=G&*!G{gU?c##K^>l85ni8*Tl7*!-lX30j!P#l#if4Ar@)Cx&Xykc^OkUThPO5QZg|M%trK_~d*wC7 zzo!~DG@h7Qx%XS%6+^pC)lZ?kZTrO2N87wz0^YjZSP*ZirWbl@lHcV`a<=q%p}h6@ z{rw|s-Yx}i&%Ig@Z_RdUJ9tYz0rQ@I{_CGR-{!3|cw0AWS~2fw^WSgH)*(NkQg51g z+WSqaSnAgBcITEK4YYad0^a7{QV?%3)8pDyI{Bx&^1I5O6RCJo?Lw0$8*dqQVR-8^ z_o#Q-ymbX{!?renH_ept_TW-N4(^w`Jr_^XZd3Qrzl4GjCFU#&_`M zPJr{_Sx9kuO4{(&?z0{DwC}g>;O(aE8F{m6WoDyMdE99?s*{+y8WMQ3<+^Lb+c86r z`=ZTT5Ab&FRoCu#PjfwmhJO-W-ZYd-p8~F>F8K{_S8sgyE}OTW;BDaA#JpKO1Wh|; ztu!Ag-ntp%u3g^vRcYp-@lsC1Tc_uL_5+)@Uf^wOyBRy~x8`9_I%N_Ps{Oo$X7l(B zZ@s?WV`rPU-r((_Nf~(yos8s{;+}m4-I-FJ@+mZ`y3|!3@oc@^4vaG^Z;0>50+I&phQ*kiMFCoSkU2<`kqne%`oKYAGAzmf!I9mSazP z!{)6oc$?E{rvE)mCgE!`vwDgIS{8n$3H<5mNLba4f>eJ;-%AkpBJenOzdAODyX?OIyyfFhc zS7nGB-oE*_lLy;+4-d}?^)-6(VkJNeeP(Fn`GB-m`a+r-b?t*7QO;1T1-j+`| z{Q{e}e&DTdms#$6>d9I2Uefrj-YR(*XC7*LG@5IvxmJI_ag~&faY;*g=%3`X4d#a^N0GS z9`w9P8I*Cu+o6NY-ZS&cAma0^U0Hn$tSoQ)};a3Gwq5nys1r zX?gSP)bRG9$KH+~@Lc}&!cg$GX8dsC(|AYg5(L^OmY&=uT7l3*~L+6W{Z? z{hkg3Z?`^Hhc`}}I@L59m8X0P&92Joo1wR0_{GMJ3-Ucr!h&Rn%QhB5kVSYVhTtzKaX;fV5&E-vUN}VhH zhPUOj+T#akm3bQh-Znp3hc`_vjYryx6wf0kYx#N8RHuiURhmJ&yv20o*{R{}kbmsi z!G2Fig13A6&CB+ldg|8wHA3jxP`_rLe%>@0Bu^gQO0_icw7hvL8s6sY^Re@6-bR79 z1xqvX#=87H8ESe`>Cn@Z`bo>1RE4|Nc<86*T55VE?eZqgP>M^Lt+?Usly{$fkj>j@ z@HXL@8oY5w)K>nw_gFLgv+)*sBAV%GJe*``4=!)aCTUGqRwstHJHE2!2%EPt;H~eV z`TF-%>OqRf>_u9eDxQruR_CrIO}|Em=AmE8@A4+CVw_sa%y`LfcspkAW4~ncHWs{H zdR-0PSc{tZsp+BCc&yJz<<4XtYUZS79*w_z3Uo39GgH$~O;0KvnqTs}yh$BO88rUV zTEp9r?{537&D%Kew(a>Eyeacn*pN<&am`*d9!+nOhjI6Q(^RL2nwdl6nzi&$vzDaw zyfJeIZFqbBBj3H)=50K9dw%Ew|9eViLVIZDwJvXY>@IWy^izlSMt?E9mAW;&ty*&0 z-)-I|fVW$&ufZEXDeRn@)1aT4e#Rvo>SrFVl{Ee3Q(%NzlR@%J@z9)7p3rzPyp_5# zy#0Fh9~arYT?O7&zg&YiX)>X4G8wuy#;>Uxf4{|aSmfGbcq{G5@b=pezxq3yx2wV1 z)Da8)?`>BDhxF6KI5j=gp&pkvDVvm;acb*xZFuYb zyYna8yiEjelWz*(jkPq|>ZkJS3(eZRJ?{P1%-zOJ!{sd|n`f=z?WJE|yus#e5_s#o zxdv~XyreyKD_n6c{ZgEsytT_)9u=Esr=%{AotAPM-bNqs^v7-9CWE(^MlbTer+H6I z${!jJonbM&g?2^1Gy!@vnxwhAPF1DUiQ(t4MKr^42bInyxe+t8;U| zov`btr`fzs18;BKo{=}JR<5@}hP~J9hk4?8%VPrUkeaomI6c%ZZ~SU1UFyp4cI54! zc);duI(XagMhoCA)l8Tt8*iH5xndP+uBE0Y4_ypzrOpj+N8kU@(>8C{g16OIFUj_v zax$TxIIeA!&c<7#x{ayp@|M$i8Qwm0iR4!CU_hOXKgi483KZo>c3s0cQr~)brMC{74g!;)U|I?U@12 z-zT31-d>*^z*`=ZmvTyRX{wBC*wgdI%+0K&i51G*<>#;KY~OFQ!P^7()Zk6?-7G!S zJQ2nW8H_0i*S%up2oA>nLZ)`c-=4}pmyX%r= z`u9}&1hI!yI`3Mo?|zde@A8(@Yz=SQKD1>wo42{(ZQisRyyekPOwO3K{!<9e9O{>< zOYuDNySz!&W8$UG4R5!+HvCkZw|U@g@B=k?i>cdE*0$XHEtJV(cq^T|;q8Prcb#GL zHXppT>$+V3p0*U0G<}v&A@uAtnVZd1C~wQQyyZBXw*}zs@fme^Ypo~Ja__fLo(tve zg+-q~$L4Jzc)R7nI=qGMuBG^Q@Roc6<~{xSpy&G8ye$H6v%0TneebDsCo`UW+CjhD zoX4kcJjI3w0TX70lcL;9p)*Y0wbPD z)@2}7o;-RehPToL3~wiH|6~=noyz~8>T>Wl>hTu9Tg)j)Ya4sNNnPa;&&FGZPmtkl z=7#rmtO8f&Z3TGidily7?`c~0GMrT1`!j~aY`kTt`0w}hpvJ4nyO4^iU{o&#u|| zPMfz?;B9XIRr$TAoF2926f}N~N2*mm1!ka@R%zm?@@Q&lJT7l}bXdx1-qTN=`|v(C zZ`Xsj0n2jaO*4%~dEEQ0QJv&n*X1o$MT=X*+rOs1_l7EO>v!4t=zZG!^QftPR`;BB zSn0cl?awqRZq`=neuEP_!X3CJpMb&bv+e)%WEHoa%Df#@j4NW?lLjmvnjB%*G6iOBv{wJk1a<^m^OtnH&D$@OEsKw>8(*;H?>^ly?s< zZyLt(?n;wCw9ou_v${3++uYBNyTak^xGHZmUigpZ&6-%{CeJ&rVJ61IwZ-sOs%Uuo zm|67-_BA6`c4iB;ZSeK{j<7vZ0?Tb$Q@l_Wpo(o9&F96U(KuBE0& zqxHP8ie`6Kb;Dbiw|sevbHANb(#*+jsp(-JO`LhSmT^CCbh0Aj%1TY38R6&97O@$+)~}GH5(j=Z3c~H-2}c!`oR^-iA%A z!5h1^a@J%f##3Z1_8EV_HL7#&LeqoG8#BactGeOs!F|T;@9=hZmAAI*YVa1ycBtR# zv%;{HM~ZV+%&GDF?>Ah*5QqJCbcvT{+TbeX04P{ zTFd;@QXa-#-uP8&sX9HOTFS||v?F?0mvL%8Z*(#nH9e9p`OW=y$XjMy?eKO!c)N5> z+V`|kpJL93TIxs78?$M&R8g}w509U>(2Abj8Qykz_iIl%yj@V`ZQS%4ypiFA{nUK# zSc{r7FH`Z;5+T|@~7a2~c@(O6yN>pfi_VcF6!0POlx|GN8_TXdp;J@Sg_y7O4Pn)** z{~Ydms19(NhKlrX^D%X^0T)v}W^MdViJc}}hHedTa}MwDmg)s({{Odq+Oz|2b9&t7 ze^52OHG{|cuvM-PskdyrN!eoJQs<#@sRxo))>39EgEDS-d)qg@v(DzNJ$M^Cw;o?+?Q*GWlfVUox*5J*PvvOWi zKP@Hh=S|v0OPowv-Z&X+8iu#^M;+ks))BnD*!%X@@t$%HlIDD}@g`+%=6HU*S-Up8 z9W(!&PucfdC-8R9f*QPOZfSaWj#dvf??bND_$3eXP}9#ibz0su6JT|A7^<0vnt9ky zp}cik`z!o`R{7Tpmw>m`Pt@U!opVPT9kZ7CrK!@R(d;KJZ<^{wdN?tcH-1&poRzY3 zbHA!>@3%|A+l(vjaNkp@meiZFpPF}F-fP)2Yn4xdU!|r~qhr?QJ#CjasiJ15 zp0$Rz^QO%^#pbOucpI^#25+qE$=?uu*jtQ_zu!_#CRHA38ZK{=Gcs;?dt%;2d)T~n z0dJQ)U5B^G9&%+e=A`tz#Z=KuFYo+%-Xc3Oyj^g{V>4{tx`MaI2i)ntr_C@WO%I_S zzIuAzVydvBq&4|TLTDbBH)de<&^XtYwBhZLKksJ!yxo1;Tn66OEziiCRV&wfOjC;| zqVbncfvc!B8Dc!FMXkvx`K7qao2E(zkKyg*O)IL`+naxW+6}zTey#>@Qs<#Pq{r{TtY0^W@9$en| zRgIQ1Yt|O&DU`Q;&Yrw?^@1z^{#y_5wryn#<1N)ml~19iFcY)uLU~(tbnh?Oy!8Zc z8((YyZ_+8Jifc|Mzxz$vg{CWwhxw_cwSL~1fm+JuiJR}=maMp8gw0zo@V0dLUGeW} zWS37 zZoRnyyk*!+!Mqj8Pu{ikyhWai;ceQdZ@kau?F#TVYfCHSO*%ow{r4N4p{EeDR#Pi9 zf1$j+<;bC*w|VOa-Ug1nyZHB1Gu6C3e%|t~US$44d3*ZzJ2`(pq(69TcUuD9LO*e- zrX5pv2XFcbDCcb6)6ZS;3Fm$r0Nx(mmXSAQ51GI3N|j&A&pg)y;hW zFlXM{&s*NrHThFbH7#$R&JAy!kJ)^x&D&`3w&b!0n*TlZ>@hRj&Cgq^UB~mLA;07? zydB+b$c;8{W5CGBqufxGamHN5?8?aAM=@3(Q_?a3bN z_3tTvgcvpZkv?&fpKFWME^nIZ+y%9?R+@(7&m->g#?Ccb*_Gk#{nNkV{QZ#e;O)`5 zHF(qft4yf}5C4@rN?IEe53TFxEi_w9=c(38J-fW6s%YggyuESJ&ExI+Z31|^?$H{& zS$Ud@lT^L=c}rD=c_bd_$wL>)+n7UNasGVaD)4r7pAGu=RGOF+*Gw;FttP+ZiRUd< z=c)4ecS`CcXJp*mZ-=aU|BLqhb~SkGzVJVqH=Z?hct1H9X*PhvFjLq95@OJwZ59;4jD}T*T zap;uzD>!PY7CqFV{&?Or)upad#n}Tj^Xqw&GBBH@OKS~p%X(}--{x&Hcw4a~F>g{o zjE6GGwW)MGZ>c(oIb}@`dfqhIJlz`JCN0`yo6Xx4@HYAB2JoiYR}7P$nN;rNw7h95 za@X`zOTQb&wbWcoO^=^9X%#alwc+hoop#w|^EMT{T{-ZfwC`zX|B_#dbMG1^r967x zqzvpNMspXeMIDph&zqDvLv_R3bqlX@?zd~eTlW=-d6W8273b;1jBVNN}7(&U*<@-#!dP~QG@de;MO-ll`M zJBB=*_C1Z6aFHkI=PgwynmnmI(&P)}?e=?*SZwolEqI%?st#`%-to%!t?mSx?GsukZx{aUaOdaO z&jxR6Ui}Z^jU;5d-Hl594|3w7f~XEA7JEZ@W&}{cig`ode#cje5lYp5{G` zRQWw~sCgr0<1JOUMdnG%o2PTb+n~k2Jj3Q~E_fSoO9I~b>m6!OZ#5YRT^q_6`)Mq1 zp*y9YnmdxT%bS#eaY;*6N^!&6(L?Y0q|Mtr@YZE(0^U45)a0re{Eg);j|oVhK9{#V zs+V#a-hSQp2d~+@%?EGKj(xOsyr-cvY<;{*6DX9o_kZk5@3eVa0N(DptqyOzSGgl8 zP7igcpKJ5b$U{IJd2Lh!bHdmY|9`J48w)b!kP z?>Eh7gdRO_nz?yAhPS?dKF#^{!Xof?)m4wBeNQ7lBNc|3hnkb*-szFzT$_zIsR}D< zH1kkv)`oft<*onEKXbf&zbyuD{nsYujnm-yGTsb28*iyPFLE-4@^{U%L}pH^y~RLolD(P;KST|Naym`#e)BWbP;)lyEzH96_`^TupkMNN-KoA3q~`z_T3G;^o0LCwQ^==DfATb zsLPz$cw>gp9mV)#>bks1oy1frUEjE%J2g zP9Rl&m$y`zt!WtEPX5x~<80op18?o`58zGHr}X_Kdus;m=Z(8a)oo19n6)l%sdi-L zF}$_eeacXqx0T@Sna)o(=X=V%kaKE!LUs8RQe_LRo`+xBOQF1-ee)&G&l6t--qu`O zhc`}!pE7EC^3beBt*J#nwf=tNDpq8inx1CRe%@j_=h~7s_uJy>Uv>U|@Acqq#fAXh znqhjWdhqj>s*03Hx{D-#Ox({~OlE7X;qAp4Yxk; z)`{Jo@;|3m%DUg=Hp(A*fUN^J&8`c@?d7k0eqj~2oyxo&Q{}Dq?8LnBij84U%A=Vu z*Gf7YZ&EhRPPta&rzd8upSPII?69QG^>*S}CoFP!JGRPO`;CctV-FfO=r2<1c`I@k zQmxQ9C+YGgW#cN1w(=O>M!j|SFFCv&SLN-Yo}1#|Q`V){^h{6aTKcJ(Q_?PPp&6vi z%@)s(H*2Scx7ADk)ZXFk_$qHp=Oy6Hs$2JaJk7~U9>%FPJ6PHp8eyuG%2um5#;JF&{!puSK0-&2Vz)|GV3S}A9$csAZrRoCQ6H9eO% zO?8dO>d^4^i@X1Ct;5?%Ro=EOtizk8ECL5Em9ZHTcakEA8*!q8Q!`r9{RY$ z+sRel9)7X`ym9i>R{y!OT0C#uUEaH9{z7@X_tsS%9Ntc;@^)RnXWaMHdY5xO_n2Co zDo<0Rv+)+wt<+V_E~K1{7s}g|4g37j;q6~l-mY0%gEw~DT6Osp@;>WS)6?`Rc?#ui zTgOB0ba*?p%3Hr@0(j%6TbiWCBdyINF7@NyZ;~^Qt~5E>54C13J%#f2g-L_kIJ})! z<*mb@XWjRd-AbPtdK#r;dUJUT-9=1w&039zJxhLd8V z^-$S$Xk3#s#uLvQ-#F?#b{d+K{pfk)Dpus2sp%ynG7M zDmv+*rYDtVero2I;`C7ad6Tkf;+DtowrTUQM;+eIsq*&HmCyU%Q=S(!chnNK|9;c# zRO88GGNE<-yd~_=@OI}poqy`^c5aooyKbz(8@bhJP0!53i7`$cnlp55w)-vbu0r$E zFHImVZ>(Zz!`ssfhMey3c3zdYHLum+jk{>A+JC<_^RD@6)%2N`H%~>w+sD4&W3t2B z`BmO#j()-ao_g|EzMq#op_7sFGd7F9q*rfs#`i$p3qNaq4(PtUwXvl?V>7gPmOyq+k49I z6+O_ibaS9x1^dq&a2V$?A?xz^>4Uk$Ao>SwLIbv$qEkUOO=c?@qS^xbU%cr!n*ZQnL+@&7r@c_RQg zt3cgfFtE`)bWA^f;9|Os=}M{_lQS)FQir9u0q(Aq_l>LqSLUr9cpG~4Oa2EnW~ZLs zGPBBls98&j)02%iW|p2AdRUQhNz+fQ@k<_;H#%7*QXAg(z4IdchDe#W_TcT+wRL!l z>>)E(QeEkk>DPF&@utblPN=z-njUI;sOh1WJT7lsC23|;YQx)i2JZbn`+n;H-Zr&= z*?+%D{p9o+NzSL#e>UEvY@zWYrfZhhPRhzw0p?rts{85ZE{B5GW1iIC4P>W zUy4f}jlX;fbW+D;=Gr_o`=pj?xx8@|wUj{_H@uxU;#V)(ymbO^*WJ@Zyeaop;Wp-$ zXzcwKx>Nc?ckS}VYzejD?XjOv#9v4%fB$v~c$;|1EBg18p9O06e9?e2D?Cus!?o1( zu~G4cC~ry4Bq-Z5WpKbCvS{X)01kC%+H*ROPc=j zDbUHUYC6|=m@`yMy-9f_e>UEvY*M_`mErB-2Oqt_=B*2O>(upC{d+3yv?a!~-EVm_ zCQUM?ZZ_VePCRkL+ebD${0p16uHfy-89R7;^AGG}&eU@XmG5R7<;jmXYhs4CEu(Hd z#^&uZ@V4f`9lX`~3nGo;Bvsw~c(Xb*yqz;~{?j&Z-N4)I?yu?JQ;X-IPj1XwYhOWi zrM21KQz=_B#H9&m<4wxu*{R`e&@X%SuzBka-UiPph&NB46@D`8v9eZ^pPtso8+S+T z^2Rw+b5*D=}+#thV+iiWotEAHuS^VSQzy*NK3Z~SDjpHw>U{4Q^) zI+ya~y(6h^THd5xl;VcBabNtp_4Bj#ZPOdPt$jQrZ_@5GanEO@atG2Q<%PciwOF z)(^bhyQ~Ip*7t!m-#(3ppLF^er>2LR9%{+MI5pSmr!8e*Hc2;XE&Flrw^UtOc?@q~ z8uN#PZQlBWx7(ks!5h2PXx63XTE?mAp_V+n52@*wH1lWUO_P~bG+Nq)W-UF;L+$d$ zRT{0_wc+jlAMSR8&D#L*wtVpG#lNSVhGxI1Jglqdjn$cfnx0UdDvwk*o;NA86*s&M z-2S0UZQcfgx3Mc5z?;=)&G%a4i8-lkylJ{>Hjm3&%-lS?GrSG?$*MDL-UflUZZEcA z-aLD+%W(7amWm3Ox0tzk)*9Ys59osbaIpO6^@G9N3&XY*|DJk!uFFZpdm6F|4MYHanqa zw;CO@R?4rx-=qx6xZ&+9Kl|+uZQh21xAr&J;f)he{yt3QE}~!Ab7gI)UsH>o@+mM9 zS~1k0s#a*8Y`le5Dfta=TXsIVqs`lu;O)iDb$AP%s>V--HU2z2?)}ybUFEk^QWsi9 zIT^#-_ulqD@3(my0p9K!^G1H}sj|<^cUtJ&=r7Mdw8Kz8D>h1#H1|`c=}Ou)<0Wl) z+iSq~`)uAug16bX2JpuGnm(Js!=9D_0Xy2zZYTRsIwq)sFcS?M*dH|#5Q|~(acXBlT%vj->I}p)0M^}wY^CnHE$X#n{70TQ2Pxo>Def@awHf3E#-ZVSqiG=#OR-;4n=f|6-Tgel; z3+9p5y1YqFX3L<>{Wj%?_iVNAw+Y~_SI74L_mp*W>a*@W$sSVa@+suKqtN`Je$6xr z! zc;o+UOr4>JhTL76KFg;-Cw~=2O^>8Q{ZbyrnUkl$xa4QNP~J}b)aRZ5KV%|!yYbQv z`uCK(XzAMJt)=fs>c3Fl{_w32KV{!PfX63wT1Hb++|n2V)Hf`yp3C*kvGkR=^=ZJOFGmq<&okpZ;~@rC(J`F)nz=? z&$TXZG1-{e(&qcOhraj32%EPl;O)}OI{M#JskhKi9R1YnS*jK4(fD27q^>k^dZ^h6 zwd9fFTuUwaIZ2ndGw>&v%tp=3lBU0;4R2ptaPF^d-ll@LmuA-CjXAjs#;G+PjXyLG zb8589n`Tu@^tiln7ol^epSqOO@V4daOHZ+Ry9T`7`A|mQtXfU`>rl>?+Rt0F?JlMt zKX06gMteFjye+$RtMlil)4<#89-aK}sip^w$J1v+SoQO!*%3X>wmbHgmN#amE=|Dj z_G+g+erw-v)4|)&xixs>d(E7TQ`3`5OZge6mg>?&T|Nayn1TFA9{MGZCXcjM6PG;f z&Ci=A13lENQi>biwyy2>Wt+Ea!CU`FYw*TAtyQPJ-P|7W1Cww8zZcy1X@8hnjvQ z592OxN`69TYj}J0`00ym-sXU}9!qlMjl0fikD)WE_+8$btwZ(`Gs!}E8#!`?^Y82D zg0~kp)!>caH*(jKrk|RA#;K)AN^!20@-r@JKX0reX=bJ;G|si$HRDoFdZ_8o#+#Il zaZ8)~?TFD=J!9W*^T69P13DM;o^k@sp#8ixLx-9k$fxAb#+xRyRnhP^y7TRyvU!^i z-tJi5VBXGB{MoU!ugaUKiA!@UhBv9YrgM5?@?_(U)s@=t_UE@;(9h;=0eGABTms&# zdo_p$9n0`SqztCT}Hecsu0r zv1i%5Ee3Bdy;u-$Iqk9TJ8R9P;!k_0#q2I-t+YGFi@D$40laB?(0DkFM(O6{ zEstqr<1IrchPMy@w%3|2^fed8tEp9XeIkq7Ka?`Q7`CRj8SPyQZcmMzbz;Oin#- zkvlT?+p4z@pKtTF0=zwUa~CpW7@n&^mc>DQZpWnmgZ6$b{ zIkubsJ(YId%yDZD6`t9%pEo)++30UJogZ)3P7QBM2VZcD&D$#QHu$y#yoG+JK|eM7 zmrgk}&YaXPZ&D{*MJ;tgPiRj1shLyKl3&l8l!0+-YXXM1&pkfD`TgGO!CU9;8F{m6 z)x1MEnNUC1Qm4w}^2REubgFJ+`qA?iQ>C<1!`pzz&vw8W$V=D$XO(%JK{J16jKp5P<^XWgnSjBCpGH=IJdAse-CgLsO-s@hs zEtfY=iCW^;&zrO=L)`H8#Dqie;}_oK?bs@Bv)c7&efOLG6gXAQDvgIZ^U$F^6w2G! zJE!mM+;7KKc^f>b0le{BPM(Mqr-wS!&$Y$eZ!sOR+ZgTg7D}wLy1Cze`L2!laYk?6 zZ^u`8yX5W~ym3m@QeBNlTFbbUM~ZW;q`8*b<&9sZW@b&C9!WDNHT{~n(r@d>w8Z*H%Ys^Nlr~XryHa0ccGfz{Jd#0NFHm~hPT&$ zat{7G)i>|A6RW&EHMItBR-VePrMT3OCLWVBW^G#Dq@A)uYWk%(Ju%wP8>>@isAzaQ zaIa;*bnds4s=VEBUk%=*J2SM0M&)5$_RP4SH#(VtnjUI;B2TdLNoP)}7CqE{-sse5 z?#SxI@b>$0pL(am+sRel7If~Ve@|IU(_5&A`T5Vb7}w;apPFm^ywTYVTG|DBlRPeO zbW$@LwZ>EO8{Xy(YLCDE_U3zfN|m<(*9P#$9vY>Ud#`yju;=FFE!FPaosv3!HK8`V z?f;Lzk9Y33e^q(gwmyKjJf;z|-_U;O*Yn0zG2O1;>36N^*6PaecJuj5k9BxE zwaVN2%X<6YQ~qERYU}N;d7scj%{>-{vX7f}uydB&5;x^9x zc1D%AQ9b*(@2Mweg&)bomxX?4f)LT*a?aOBv{)mi$tjYc*QR!#K5` zH?9ipq_o!XHs~jRT)oL8gkgPd%y9ktx4;7)9l(aXTw{!Z(ek- z!`oR^-nKm2N_f-Eik|#<^PEWKbzmpdF>9HJIxTO^Y-z*WnPY}da(Fwt%G>%recktz zc{Fz{J=Dx2Y5J+@XPlZIYI>;YNy}R(CmKIHkI~t9lPYqCmNvZ2zU7et4sYjFd0V@% z4sV=%r{#?uYBal|pB+-u6RNZECROotWq7;)4}X2k z;qBZiZ*!gu;Eg>;n?bw0Njs9d)%Z&jFuc8c%Xc<9 zyq#C&ZDha8{qJdL51QVjoQyLMH9eu4d0gH?v(eAYj5kWNA3bkeMJ?^Z@)+K}*Kw!c zIJ})-ocUeeG^?0_nro>`9>d##&s}+$!`nqw-WIQ@!y9W!+NxXg zQ>O7SKc~%jD(&(nWoGBpl0T1lOkJ0^oH{hTo&L^;M?1V-T;*-@^9gv1=|2yyG&#$s z5Yu@}Sz8QmrD+)69(&J`cY(JJ|Ni&&ZSns(Tt2j)|3PgjV-4-IIf2uBRI=@q*o9_L zc4dG&_0->5KmT;UHtoRMmQ^j7H{~2^KJf9pu{w3=Y3A)GY5rYD?!xf4qN!3(eLHerYl;Z#i{qcspbB=sEWN))~Cry|o5!-lpGP5N|vM>d?K%tYxkIcw-ewa~IT-UlW(| zxV%YDO%;#F@b;}A?EPQ%{dO66yZnuUc$50fY44R^IZAc?yfvz;JnFioRUfZO+N&89_&yTlMCzy7pq)wAL!!8VOhxOkV{{Xdozx4ob3)>Ga{(jT! zSMr3;LGtIvo7A}!mpYXEp>dMt-!(gtw5Ph^ZSvv$er$jL))Ty4Ik`!B<9@kE>(}6w z@6)6>*HV`!AG(SesF^2JGf${yEv{wU}!1Avb`hvHm_wV3s$3F;6GXcZf0d2oC&c5F+2XEuM40Yergv?hyFH%k^9(rGe z`isBcLOV?5ci+=gnXNpAw>~F*{yjEtSAe(fGivZ=<*6{!4DnDtC4Y1BCUqM#c|UIn zD;nMo+UMjYHgEmF+jARQ7;mAIm*(l`O28RCYwU!UB2uFYG2 z@ODSHVeWgHp;lAfftvpPylFC|^2D&D>A}xi!s>>%&U^KI&gN|Zc$+i31@jg&D?M+a zRYLt5W@0>k-lWW$U09xfzo)y;|FO;6K=3wXV?n%Gy*2zk)bl2tNNBuK6UfF}= z!`tHh7dro5W)OI5*K2t3?`fkZ&l#rWjXR=d#YX93cq{GN@b=EGKlxw#ej5zlHqJ}T zTeD4DI_0#yNmW9p5$e~}b$Lrw#j0+2>p5!VM{M4PfVY*8)!{9t-g13U`Dd)uoR6P3 zY1hoA@vvg3W-aQ_JT7mc8I*p*+tHW3{b8H8q2TSBzE>9ip0a=G)ac1W`+3WwVxw{v z%G)U~zU?}jw_)IIz@i58ra1-a(=Ej{Im@RIlR;C3p1f;G`LppRW#f)4ZSJ@AN4)6V zZ^OacB~R7iEkoU0-%4wbnQQ6qx6lble#SMLJJtB<;o5AxF@sVY-X`pSw8Pt#;B9li z5&rkI$oHe(~rg@-SC>V8jmKw zG&etQ(kgbPsZPH}vo1AjNt*fT$;KPADYfD4t_P3)q0QSU@HTGHNdJ4PtX1<4$$KrQ zO0B8u=S`DA<6&3SnzgAsh4S{beOBQ=C@=qg%hBMi?{xvZ@e`M78qAX_za~#Hym5D- zTC)r0l>C0)q*ctU)aHJB;LxX!wC}eu;O*t->+qJapN8Cl(yaWvwZvUZJ^Oi+<|D`oz3RyN-$TJ0EK1wDw+;ZLW2BlQKwgR&NH)-LW5+H#(`gO41&` z;q70u|8SN4o{k4^qh1c+t=T3Y(~rxWG~t#WcX`WcM~1gO@BXm!{|!z6Z(T-=_P?hw zXPVQ#YTjL%THH(Vys^5bTY6&F7Rp=uoqxKIeZO4=-ZtG>hc{{0F%w`;NvFy$<#BnF zoa|8Jp}!e)F}#%~WA3;2KmW)rHg8vhx0_xI;EiuLb&*pItt%1Y=S`}@Y?3E*8d4tC zVou2~#a-T*Sz5)4N^N*McH^M$*}P2zZ*xYE@xQ0aTAAOtRQWkWYAL_Vo8-(RuIVsU zU6(g0b7WVBx6Zpg@BDxGlfc`MHLZjhljq$-Hr_(nEA7JYw)QI*9$@n}1-xw>H@30wY2MT3+^C!Le#>JT z@w{fcEjxfyzzTWYWBc5wd7&ENbTp1ol_UtVX9i$c;l|A zrLIaI!`sjI`hABgaAn@E0dMo)sKc97i*b$i^jUeKQsrSSDQEc<_*H2Y8(FI+4=vX zr-QeVch%s{;0vxnOL-WVv{cLGjZP^Wdzb$#xB~3qd z$^Y;BZTUIhuz8yW-Zoq^!T+90bsH7uRAYK5pF&KARGq~1pjqqkrdgH2V|aVdeLvY^ z^EMm2t(caWH|Z3lxU?^gU*nPT$MYs-*2H5vXMRndVt6ZcVt6~{hC|M@d7A^?#ypUi zH%=p!)=V|DR%3Y!?KZ}*=S?$bkH_%V_3yn_+PuvLZ(X}y<$q7DH@fC`q%qUr+B|AW zIm@RYeU)9sXia{}BgI4e(eoCX+439S_W%9Bx7fVR18?D`+1`?G_&P5ydBVGzt7sdEdp;79;v~bm8Wvn zttp<3x0W~oKX0jaRQeOGfqD>JsK@dCZ0DBy!H9bm5a@y3Pb&I**G+l8Q)S9&#kIP#g zRZ5*0-hTDV-G4D;q3!2 zu5;&ZApRPRWyvH_fgskNNg(oR{+i_LiR^Cv9H-1CGT`+Fds>vSx)S9)S9w|TjXHGwF zbW*d5l$joBt;WN()Lcs~d5Yn!G!4VsSGQhrzQfz`Ro-U3QiC^XlH5^C*X2{-P9-hv zlzz#>xa60Z;o2Bo3~!|l4R1#c_*q+rw-c(o4H|ik{ymM^bu+Bx$;RJr%oe&k&038| zQ#YPBsS_)1czbZ`kS{vComl0q%gu>-V-M7t$%Oi)oYGA~e>`u@mWNI?4VO1&rjFc& z;q5P9Jm~WdZzolGdv$XS-ZbaM-a`Fc+bErlw>-MdV|vW#@|MRgN;wU0x4ib~e>uFJ zT;=VaG1IcWr`%Ca{nz|82PvsyxhD3~!~*4R6&z)+|Qe|3&yEEyD+>>nDB!8{oB8)ybah^gEvpk zns-i1^y_)cdv}`C)Oe&m{k%!5GQtx^@%(tR&c*OH=Plpq>F{=1mA8lPsKc9;Gcz-(rk5&@pSL`Aokvd1H1gxks&07O z`5oV10w;bX?bIHYN@&=&OFqiYYXM= z|2^LAE{C@>s=UpfSc5msj+)IQeMY1{{k%!5ScO{h$HY_RFO;_q+fT-iYkTwWhn!jE zZScAV@Rn+lF{hc0x0s5JT3aY@KYgjoVb1+_R+YDQ9cQ@jX`?1nWZfOSC7*!#{_WV) zm)zs8J%33X5r@hmfc^ZD+5_WEQ8~=g#PIKSqu9{FYCF5oTtb8R(&=Cv+|IP>1e1X022!8*frJPu%eK_IBM*ba=bC%G=Byv)%X9;~FQ{qQXQ&$h1CY#4&fSY*hvzw~Gm3eCi-mZ9b2XBdg zL&S3;HJ^%5e^c{zVCQu=*u1p|Z?E^B<36a;1bE?6hfaX`nTPREt;xea)821Vbyk#i z#I=%^)@tJM_Zz!nMaE0o+;2a=r^8t`ZymtfQwsukW3861<9XAt)C^OK=S|v$H37rh zos;+ezRg=l@V4fOR?eIC8LYf$@w`c=z=|3V{nT8`xOK{v9Z7yYZ>&Pi%pPrc>vY-W zlWg8Pfwv`B%+_VcEhfYg<>*6_CM(kp*y z^L7b%o3z^4OvYlcs2s`<2T-gr`l z@^=6A!w$50>kQuB7%(r}dz$Lp8Yhl0tMwTSq zu>0+g7ys-Go42mu?WX5)8{QuH`PYAJ^VS2r-8Xze@$V`7kI~ZJH96yX(^P2&Pg>qQ zJ2JdI__1C`+Pw7yZ_8HK;EiuIwRFDpuph>$>5+2ML(M!~%Q*ei^psD5kx4Ve{4tyv=(xfH&@rn!BTxJeoK?t?GuiS!3Ir zX7hGAczbp1BK>pEp*QGBEDx z#PIgt$F9LY&?^7?g)6|@6SoEM#=6v@J#%f5nti&w@vB94CDlsRkDoWGdWN{+ZSjh+ zAF%JYe&Fqd*GY>tCYqXv>R*|%)qTy}L z=FR8Xy!8if%dT3ie@{KN>T=R-Yu)>;**fI?MjhIxo;PM@MQX`YiW}a(^wj9D+q?|` zZ)4Wh;Z3T=x-C&RKX0j~nvJ&%)eUbu4e)x~`7TdfH0&mYuDu_2}8jSOM2F9uB@$9|s`(8c;W@8m*rlu!UvzA6P zKmCkTOCFauuA-JQ&_iua#_)D=g58@-y#sg{&K zo;NA8)rsNl!fT%&W#4bZz*~>|GxEk-%KJ8mNli}W94d90vwRASa7U63^-Fp3h)dJZ z^Tw{Eol5c2TEpAjhmQS>&D(JBwzc!J9q*}o*GXr`6VZ6+rT_ZBT=Nd3N2)8uxt6+o3eqYm&I}R*nzhW6xAyZUb<4Pj z9X4;Hz}v{#0lX>eR+yomy`_pXCr_XGCGGMiIa!5T*q~VQNyIh zqp20+iRX>Gpk`O4xZ&-$Ge7A3f2tF}TlYn+oHuDc@w{=`&8*{jlXhp#*6{W}6HeUO zzTd6_Z!bO>z#D%pEcL)R{gRgajOU?U-Z(vWD^-^~p>gJy*3zTV+>4(#IyIR+9>d$_ z{YUO;^L90Od#K+^{d*cZ4b5I8j}&JQlIBe4Vcg}7U!|5h3H8t)s<{hl=3$)L<&8U{ zlbV?&ZTSsvKmF@m`~a=;-$$AV-qtLw!JCz*^1YT6XFo9>uBB#vYR-*Xv$lK+npK)? zl1GYj7kO)!H>qN1x13%{8{QuM;t9^rGnxe67CuvhH>sayh;xrI{pjyEO%+X7^iWIr z>7kbVQrzV&CYw~H6gRwGFmTZB_Io-RybT+)D%*Ri`7F>wE%h8a2j-+^ekoo)1?j7d zQ%fC6adwhQvmYrZJuYw3s?a#Al-3&FetyNbo!{@B0^WLE7r>jQpP0#O)@nSOKGWWB znu?Mq)sCe6`SE5|G`!vQ*CW1a-)~dF+e^&msYrm znMd+S@oc=YA~knGU8-*Gw;!Ci^G|Hvrh&Jm*Vo{Ud0MM3pMqwR8c#D%DI0H^9a)nx zymeXqN9X6CP6uyeU;dBgO*7S&?n%p=#Tef;!`o3G9{79ve!CXDbsuqq`<^z#ggyJ> z9)mPrhG_J|1@r2e*%Nwg(-5TCbpR@7>o3|O@?b#axc$27IYsyv+n}n_dgx z&B`Bix8%Ln`mELM&E+lc4jYv}o;PLZhPVCi9^(9bz**q!w$ZDb^F5XJsNqWEae33M zlDcZ?hJ1?&-}x)Hg9vlTmRPsc*|oNjmr6-%v(+~H@tm(&aEA6-sXb0?c;8Ye@{a{ zFM0bl^Yrtk$5<~}glgubmh!l~ah0T*jUL7& zO}|D+<&h**ouWD~oIY8)I&wDaJIDUvgvGjq%>(rWj*-a*ZiQlb9Z_$xX5Mjv5;sr1vUS ziXtFQ0g>KCR73?U*cB|Os3?}-9yiZxXZ_@6hI5T@Z zhPRI2_|^V4ZwtWNOOtEx#{AUMt{M0AQ#0@ z4sCe*@GE~AWAnBUysf(@fVUhyG%;4t9-H!q_EtTGM(Zkc7aG6In`RX~$|)J%4mqsb zf7rY&0&h2Vyw!bAS!-w2@%NkZ6mow?#7v&O#q*}@#PD|9(qlhp^R^hgjh`0CrmgX~ zyp_3&GHd1IEk}oM-EW5t+GO8vOTpV?m*2MQJ$2dB5S!yfa^L*yhg#D^sl4?&c7^l% zxR!ypRoB(wjo)ulYj#SHv{s69Z4tV93Q~3HOzEeVI;2O^TuTq*F*=?%sVggP?zaox z_A}?_Z_B~kya(&>7PHeL)=DI3YH_XpPU9*~l^9PEwc>eG?#S@A-+#XQ`}TXf0=$jv zz9#K^YB68)1<`nvd#ssXTFbgFZ(Kzu4&;>Ie}Qs$UAyQ0?Q zl>F?O9v3&M3NvUtmfzs^wLRwEk>R#mmA4Zz-X8CDd%owCwYWRRrJhs8S(lpmrGEUp zF#|QTNt*stnzcf6>Uj&zY;|RLJ7aFAJ00Fm%y_%w#sJ==GY#!i@`vUR^|P+a8$T*# zmf~DX9qQ*=>XfclLKyB4+;;@n9e%>0n^U&V( zyhWa=;cedLJHG7jc1p(EOY;MG3*BiW`x}$D$P=&mnXcz8^1KXh|F-90`1QRzzF#;s zW!k>ZDTfE#5BK=Md+9J1ygF z&A>bK?Cn&6qWw~zX?c^XG*_HcP0JfITe~y7eg3IC-{tUjddA!I#ff=i51!s~ z-*wC{X{j#bEsHmHPR$8OahEsA$!wDLtTntn^MQ{%;_!Ax##^6f3*=4eSrg~Js5NUf zp2p#=h~04lE^qv3LTz{(_uZSTkPk}nq=LIjkY`X@8^w9jpk0ND{BpJ z*B|!G;SO(SXS~gMJ^^o5-MXiv*_Xy6)n#1L`FLYCO^0QAq~83zNvm>nZg^Yv8m`>k|%J2&I)^_7Wv zOVz(74^NX?v$lE)p{qju>?CF_^MvZOym@wEc>C>iK z>Xv?b7?)0v9%(J(8twAN>eQNSQiqkbhPU7Tt<6@4w+k}fM!iylx6rzpp4lfgYe{i> zBrUC#;x2EJlNB`{jh}f!r^o!%%*nXR8$ZfbQYZ9qt)&fbdz^n5ex2%$Kd-+q_b9)4ZTj{EJRjJKCwOUzs7UieuV zx>nTdsgs${#+yNEqVjB)SuNNeM#%_>rL zdWzQmo${7rC(^1MJ2Jfe{R3Zb>+p6-#@pgMYVgJ$G@AWW=jgeKTh?XHRNCc@6{)3N z(8I16r>2LR9_mmJ{nR9_s#!(S%%IeUx9z9ye!s)pr5SItTi@fq-;}kQIB6+0CuE zRisDKtS)&NmufN2oYeHVyz!%~sL@iD%38zQwzt1~I(Xao*1xZBjo;_ceSHGpq(-?h zYWf+cmOR`rJ(8CE%5SKKMJe&i_pgXik~)*ie~ z>U7_(_cR}S*0f9eObMl>INq#I3~wL0Y{@P5{ni1z^_yOYH|c9gt5(fvb1nUhOPYR3 zb1gkmoSy0_Xq@zD7^9z>)fv}lu9bRV9{T;fag|0hv*kCu%^UiW^)_!E!CUVQb$Da` zok{!eH0eysoPeeWjmOVh!W|jjwm!7G_51J-JgyUXd$G&?{`Zs&vo|SDk7gR0Jj^NS z&^&(LLaRvCr8x6zJoJa^Qh7V*$dmtO-*274+auT4;VpD`Eyb_rP4ijP$h*$RTaI0u z`|a0XJYcHL+vVWxriU7ox6u8D`t`guGE17?^t^@6(5h~DYyXRRSJ=FD0dJFgJm7y% zt(-N#FNpC(?lE^iyLd}J0mIwa{?z8J8+d#1(ZsxIdTRtvTHd4+l;UMhz~wEpx^fqWw{QL8+pBEe zx`VfSdT(&wQ|UVcQhE$;|Nhe( z-){5P1H7%CTZ1=c{@gE!X1^K_`=p;*@>EYj`I#HEQ>l7PE#{L8w}nik9gSsp62*DOkJ*+laq2&=HG<}ZrG7M?e7tEgOI^jpW9sU8 zi|Mek*6{Y@PVe};&D&7$_U20gype5csZWiEYc*44PHN_nH2u|6V1%nQozO37=BJkY zp>a)4mp87Wj@-53?YEa4wV%z~72s{l=tunTY3KxE{E_>u`;N1>eD@nOq|(x^rFdvv zm$y`ztxgPY!w)`Xv(4Ku@V54roV@Y#g4*gKh=EjVHU0Q`(`2BBn!8|}njT5V__@~Q zjUSb&OH6v!8r~j0=fD1H^EMp3Eq^s9Z&IHvCGPSjO*qw#iq0?f>GCEyBjbj*<*mMl zU&vMc^V1RFZPK_$^S!6dHND80G~Kr@Z<@@_wd-d=h=0dLBFn(h{!NX%O13DtVuQgxUr zkF--Me>`teMJsN28@>6*{cYYxgSYipKbG%3wQAL5DV4|bHEL5`dNkVQO?|ixa0XA>rz}qY9bMi(W_>MCkqf_O)bkKQCcN%qd z+N?q?bspnkb!ygPerkHCH6HhVlUA`S$;0Z@6~Eza-xrtcZSyuByuIB13H^J@9;jKD zTJlJ7t}R;oKf9$4`B^V{ite2G*#k8_E^quOS4mpwH@qFW8F;~(nHOh)GlvaB~_8)nzb5_lwXQ7r=)0s-A$}i=S;(p$wRopda&N$arwBhZ(S1$ON zeZNfvZ<99G;Eg$VR_)$znw?6M*Q_nm!#@4ICG5)Z_Qo4`eADJ_5_s#|ZIk{z)%0dD z-}JKxuGM(xrw(09KlAu`<0@)q2-VC(ohlD=QkTlxBgY>7e>QKE!CTi^b$AP%hQ`ln zX#Di>Gna8{jYrQLGix+=N3B__@klwPc&WUNntR!IZQia0Z!bUG47_QkN{@yg`Ze0m znp>tVK;vjIN#nt4mr_rHBqgb4vc= zc(W#Bc>C!`e>l{>-==}LF^>oEX64V#TO;Jr-*1i3ZA=fP@^)G0cRX(MHXXcm>-$vW zzo*h?Zu}|4WN0aCOXcm?Z=dsFo3|O@?S*+QfVY^Fl~^tAev>MO?jlv5Qh6Klm)@`1 zyj=(0)@|C!d6Q1LINqd+ns_K<8h@$0J^k0om)N{r58iGZ@U;7$au-E>6&;h4Yc(e* zd0gH!nd#w_SVf9+E&Ys3nmOrV9%}ks-ZUBLp=K3Jo9`F)djFH1ZQf>rw@Hg?@Mh&n z@_m7_=Onc>{kyzrR!JV7FFi%ze%fd#0_sxPy3wn@11W1ZwrS#>wiyk z)J;Bl>Fb9W4&!-~x{~6YWN7D_oXk%h&zrKk;q43izxR3jewz#4u34FYH%&jOCd@q4 zoUo>!e7s4Sr8v9NXlX~%TFK9T^t^EuGczuAQi&Vhx*c)RK{jvmz+3+p>+q(mRrgL~ z9!Y!l(v+WTB|kmX^jA-TAJu4PpblM2e`p@fTIO_lV+LxjlH!u5;y1i~`F~eBzprLK zcpETkOXI(%++!;3zu)-TovI@96rG>7T;BN6gxc`d;iZo{KMz>|-nQQyz+0J9D!P`R zw}hwAbTcz2cSlW6DqSjXA3kS;^Z$u21aBL*1@M+?0*#QTd3h^y0;TfyrwM+ z2)r#H^W4tyo`&8MX?Y8+zSHuyb^oJJws~6&-X`5vhc|xRNX>~cPEAj!W*+L$JoGb9 zl{ZG{Or@EhIwpT;9{>Hu&lF~+R(5N?etYH0<7U~sEdg(%UaP~KvR2KnQrI)Kl#_d> zhij=rJ@m)lZ|sDcRU}P6wd4gB zd2-gg$5Q!o>@jygE^n!JQDz>^#9ZFkp(X?U9&LDAy7jZ?*t{(VZ;!lL5^vJJq`0&% z-T>5+UyAP>yh*!b+~qBF=FH~NhPMUZ`O-|Aw-w-R)ip1;@2Mwe?)!u}rTt2A&02ai zx_sU=yVG``fcG+*Li>Z7u(L3mvfJ*9>0r#N)=y z)Ka&Mhx)lTRLA3{?AGA6?#(AYm0fSURe3uh<89020(n#RndFJicM74aTB2Xin{-}M z+?s~r?IX8+1HZm^2X7~4yxnCuSF+YxTUbLuyu} zrl+C}ZzE2&Drqwt=ZqkU+>*< zznz@%HgsAG=8ZF<)||4RHz}JG*Hn}|%qeMUtrTY-KX0w@-&080lySq`hko@9{BhO} z-cHGQd*%KH;*C?%?2&%yM{$f(`+3t;XEu#r@<@54xF!$R`gxPG<%k>J4x4_z`~AYH z8E=nY{!(MUr(`L|-laRgDIU+8rmLcNq3NMG-mKjj-X44Uz<)SjznzxxcJp-syjl61 z&Q7!IM)Tz3ttCz%EpMI)7~VSn<}CboZ9Cr6(=*ommvo@YLWkti=p}%+^{(G4nyq%HpHmLi{`QB6Z!0AbGdKy9J zyWiM(Bkz>G#q-8pd$i%L>zx-K?!2dGX1r~mm6$hAKe_M5B67y_Rzz1)PVz%7`57;j zx5?9fw%pN4D(R6XVHQJO)r&Gb&l|I`ilimK6lWfKcnb7L+U1SexJrt%A~n}m;^sa5$ejL1I=r2e z@wWWNI=nGwj1K**5A}!k7SCH~MfyW`7wTs%mp6ViCNuL?wBhZYk5BxY!`rzTZ&yE& zn735@hvtz!H>J3rH))kM;Ur&?vKDn{EoowY-cnVp^i1rDZey5$sZc`^A?(|k^O$&LOYkvWe0B`J@BHH4sWe8-nP$Az*}g~8o#tJX>F)~ z7jO9{;F+_z-_HNj+HnqV=V!ca*j$G)NN=ywDV9u`$>DhF|(x& zZ{PpV4wpH+U6Ap%aNw)%d&>8b+R9&-x0oquYOyzJX)Qf^-nfcds?uoja(Jsu#_%?? z-!cE=@OELw+myvMcx$wY7158Lx6tbRETaz16XR#ya(JtBYj``i&7ayjy!|udt>3c^ z${QyZGkLCM&f<7uMU7@xF@7mesk~ji_r*s$yj_&>_R^5;?t3bAu9-$7cr^X{dE>iD zE$vQIQCiEmlt+qlt)$E0ZO40h;!iIc(@am}37wUcGoCjob7+UD{8B$8Eq>RYZVhiYeqrns zhqp^J-o^}nE&e@~b}GfCo;7}rN6OE5+R3vkYF138L-Xh3EwoBycZRpUZW{f+4sVxb zytTcl25+e*)5v-Jym8vp&9x&<50WPzZ#i~qcst{sGoJ%*8{hi(^{w&y93FjfCj?F! zJ%{K5$5qr^)e?0)aLQ>I;2ypCprsjbRo>cwx0Rz_cOTT2IMvXetEZrxWYdqfi}r`^ zLgUZJTj)+JofzKs9Y1NZ&0AaWHgRwadj zx}>>N>KH%!WFD6{epJ%zikf~aZg@NVGe38}e(M0l!SE0KI z^|J?;x0r06wT8EkJ@)6t_WjloyuJBaPTo9qn{EJ&hm)b!td05Q7k=I})!A($YnQiz zx;4D*{kOYrw|VOX-X6T_P4_+3^urU8v?fmx9!)=f-ZWjYN)didPK`&;Tj-9g>V~%& zGrx7O&0A;iw&0Ec-Xi;Hy7#O;n}tH zcbkk`>hvvYdgITZ+xSvVlJ^$+LbAD~wM{#$SGw)ufNyfHs}&}gm=)yyBdR^xYh)2wPP52qn%_C}A( z8=cf#CB;1+!`s`|-0S?jw-0#R*ri?kdm8iE$9Dlj7Wk#zQ}~v{s69Ew!FE zu9CE;y5Vg>tJk{N?`dD~w&3~%yh*(=&d&>KdZfRRNKY#5@+Ng9#XXZx{+$sgBlW|{ z(36%oRfK@2dz$t$z0sKT^XB0v z^t&Fz+xP(o;6J3S{`_q)c$+%625$*_sM~XywMats6r}1>oEgg0G5z>?<1Sc*njVif zyq!IA`%U)!HUzx&d9ozlr2eHi_m#@SoPOTu4Aq)$rJO~q&BvS6p(k#5d+*=(!!J~= z-fu&}+q3;T>fclTRL4`d=}x1cnrj*7v>B&Am9Cybs%%B%;RL8dYw3BDGApYa-VXZt zoXhR|?F#Vrz=8nYq<&gToO5eT-q@j}L#OTMja5Q3(_hJBczaLli|`A%s`uM4@V4ry z8oX(GpoiM(q2^sj4}Ws5(OgT-9yGdo3X+qVrLO2<9>%Gq+gr0%>Ooq|xSuz67^+#N zvexjnWb35&+xOdW@V0hPC;fXG+D{|<8}oiEVq(SdW_4?LyWwN^J!bPZ0=&&x(hR)u zGu`UB={_tv?()_MT@}$osk~iu$3@Qf3nRhXh%L>)8(Au1!ptvemp927(_wS1)`+1`?Rdp#(Xq>%KOL@}r=Ba4j z(?2f$z=dL*5XH&&6f zRnhRa^xk)V#OCcv@V4!R#JsULtDm~}B-ipKVqDUpwOrmbndu3w7~^MMYUX5|TF)C- zQL~Ci8{WqD{p|TRZ)3pQGb1i{-&0S{y3B;GC0C(qB|mdY+T~5k)@X6=NYk61H%&JF zPAaPt!`r1(pSjTHZ7g`ZWmOH{B(|lvrcY_DCN6n2b?ITA>M1ZHIW=*5B+Xhen)#WB zacVzrnheb5(Qmz{r@nry&D%Kew*2K9yjk4Vz0*Sdo}TO0O0`yhzqQ2OaY}yP=#12c zxAxENwXek&*YwP_)LhFrH9Z>5wbXjvVybA?(i6kBw6;{B0Rq?7ly zdAl0Cb=jVnH%~vMy_K3C8izMcw=tf0-lUt;>dNqT&S~55ABtE1ynX_Bdv07;_dO-g z)K>o7{AhBZM%N$C++*~8u0ei z?KOClCcs^kso4*8Ob;$^G1+3)rrMFzgP%8PRgSpf?FZl9`f;1LiQw(lH)`;fqgK#e zSM-E6{WlJ8nr=fqoN%eUt$*(iZn1fr1m0#$=;pqs+#Ne-JXG`NgrRGT@cVh=uBoL? zG#+Vf(eYAw`(%&D@E=lEzkZtx-Ui%Nhqt1q(Omicyh&X(GFMX1w}&PL@K(h1 znv1<%yj3TlnZL%fgSQv^ee4pOx2fQ5+1)vLlfEw|lk|kfxko7vb4r?Psa@V!QOYL8 zW7bMJrFdv9_kQCXq$*OJYpE-KbHBZD-BqXAyiEgdGdlFpzo$|UMZ`m=8rp-)TWE$* zf6+UV`f+)aoRM+E+y1}W@S4rrbnrH0Y6J0B=82TgTjXg5{ltm2O66_+)t~=`&D#v{ z*5>{iyz!o-<}M^%1T&#MNWC#n^%NM9R>j1nX%tf4@nbgjVE~G&$*~=32(7rQTfLm`z$G#XS`bZt`v{SUn*}W-7>s~&D#y&ZPUXA@+M75`f5aq^AzZzriWVcR8N6UYObQD zCsa$*U>??@riWVcFz)ikkA`NG)>iVE`|YT|e)sn_Z*#!ghMv8f_dV5|0%wv+`*}-M zB{WaWG^DjIZ;~^#Tgz{F>oIAcel~A6g15zU0(gt*GiI%|uclWoG|nCxlef?vY5Xp4 z(kjkJ@_6Egw{KlE*7<#0bHUrV$7}EwSvNNip1mZ`>GIYRcdhB!&zmNLbS~Cf!`s5` zM}678-{yh0&VBn7_nz{tr)K|}I6c-Ln>eJ0nzg8_r$DEqxk}QZYw2gs&^Xsh+RvM` zN{UPKulNmb&#gbr`F#%a!Q1Qe67c5fp@|2n|K^Ij_giyy5?Ytjrk3)gBR8% z-Xps@f1b4fylvc+fH$d!MvgZQZyIJ|CRq+|l@1MW{WrHd(0)%Bg16-Z`ug8f%{2H5 zX#5%vb4r@0Ko8^8^i)rQ5lvUj7OGi`TFS$IB)=4APCsv&49rGttu?%z^P_K`VDq*J zyvoLpAfq^Tz7T%y>l`-u~KY)<10CmVmcy zgZug4Q!8g(e&}a!)buo$c6m$HVX8c-CgbNVVYi02wR=AB4x6{7;O(BJ1@e|^Vxcq9 z_+8#ob=zop{Jd$p+4TPR+Pp0XZ=;9xFYZ0f(Q^|IMXxQNx1x6><#&0LoS__Ae#6^Y z@BIFHo3|C>o41wV?dh#8fVY@ANGIs$ zO2c$#9NOUat!Msga(2D#R^{!4jJGK_*Wpc5*ZMJY z({m@qnUfw#OMb>fbz0tHI_FyMf^lkksOh23#~U*!wc%~>2>$b|9lV{G@it*w0^Yc5 zNozh6xi2Y?6z5t=bFH2?P3O!kX?8`A6sJd{r9S<QALo8MDdTOx zn1TNH)RVvI9!uqsCYCBc^W@`AI)%{AnTMJkQitZJ-_Ki26>F{G?JwWobArR$$r*32 z+}c3AvHw(>r%A2nEv6Gzq$Ur{PY>f7Ev?n_CRLF#TXDnNufO*+{=1JI@98NSZ%=Ih z7vhccEQdE%q?UN2CzX!pP1%Xz?SNt1`aAdAsTps#jvwT{rxv%#-%aMH*6d62OK~Y@ zKHgY`TB;HnFCu5Dyq*8|4*28x9rxR58E^B}*5EC4N-fc!kGGb)Q&LyZF3kP*+AZ^b z?A&jsXS|Jjvq0WLUyV?|G&d;^yp#@fhB|edCe%=Q2CKemg7U?UuGf^!J;lF8h=;^YANV zYOa;y8jrNLdJ6m~yJBWZb1glgdAL@irMl8ump94Dc#d5d-j+Q1{vpo&c6P?w?8$X_ z%Tc%KK8ab&y%(X&zu$`3g=WGoZ<i_D8I;=a zHs*W3xzOS5{EW9nmtWz3Pa}J1dcQP1OSKrM*5s5tX+OJDbtQGncxWw`H)hjlsiGA( zydAXr7xCZA?D%=e1sQLnudBnGrk|GRNy}R!PavK*sM(2LJFypPugEe@w zxNW+lq`0!5raUozO%EE6o;OW4jVE;1Qcfu@<@fUznvGSgwT8C|a~FKk;q9LpZ;y8y z=6_H5)>E@?jAl;u&p7kb!+5CH^A?&}@@qPmJkm56m)5$xaTPT?XI#ome?=SKF5LRQ zQyktd%6MBds}65g&LnqKBjhZfx8~Zl%UeO67~X#U+s^L&c5%kr>_=+wrr}xR;lB9! zLM?e1kJ0Rt+T~5^iq$n9`ZZdUlOAgBNUG)MjoGNBY@WE`ZR_2i8RXn=mt?$+>owf} zo_g}P=)2D4t<0TjI7!uypSM(1DtQcV5B&M$yB*#x&3Nl}V>9qp=Cq}G#`D$)yO4Ir zxXW8i=bp8Ow@K?deB0sevW&Mak2eEvoJOej>^JDXb$LsbIdn&<{8IleZ>cI;c?@r7 ztZID)c-#2azprnN-{){g-x2-?ReIDJ*J!J`x^OTj*Gh4EG`e~WJS0gon^ZkC&YY5# zaV(*K@H6OiK4t{IosN^!0Y)lyD9Z&C)SlTg2uM-$i7 zDvmd+y5a4m33onb^VS}`y}GClZ<<}VM30^~?v4|trbnY=)@t(8Qyg!sQqhLDuS}R~ z{r@rt9oGT8J@8Bo-gr+krxd4$TAC{JP;)Kg%)_|kmpt^lym6JJnL*>BpITbWxa4PC zqvLsFSJbRti5uST`N#PW+xJ^X@V0))DF1uPoI8_tc?;bgKMO+FYW&PktyxP?Ja5b< z=}L!&x0^f8`-II~C-63BSz_K|CL`^ohB{gn_1gDazfbj` z<2r-45zpu3t;~sqPFUlwor=(7^O5?G1XL!5sz)M=$_uJ**t?ls9`uEhz zAM_IguBGO_q&Pj)tp=SJ)C1R+f4{MM=uWv-(o#;6R_Lw`Z)cvj@4IZ?x`4NbRtE6q z`C75*+$9gEL=UILIQ`W0GfvGs)b!{l&kWSeCh1G@89^=i8JB8FajvCydDCQ|hnktK zxZ&;H$9xBWfLi_Yg|6W3-WLOSW35oF;UqL?sGmLPd1Do3U_4Yyd8Bw~e)?VB_|b$F z4R43;IsPX5e(MI_ZX9)`{yk-%sdVUmL;X^p`FNAEHFDg~o78!ZIUC;YdU%fmY~H$q zw-KxB@Rp-))7vWTB{Z(-LE~ZF#^fz@7u=Dg^YJES^K@u<`{(=LbBxVf5AfFZl{&n! zuJv9g;fnnik)QojyS&9zkJ&|N-B7>FTWAK(&7%!(Q$BR`44b!};O&_)W19Cp_4Hqt z=a{*dLMSZ$JIYg}=6W>jmEKzO4oGCY`K?9G5pKn-tenk=9Ca zDW?>d@-Xi5#*b>Wvby2zzqfWj-R7+~c)Rhnl6X`0T=PzgSu4#`&s!1IrLGu{smrzT zys;~fHoU#IV%0u2Z+*bqh^xjH_nvaso<3_bR)nAZ=y@xm6DcPx1u}L z%tzKvIHVC|Jo?M4FX-8#pV*OmPCQI~COZ^we8>>^ZlhElg4>fa2+Rs~PHmOP_ zZg@Mh^DjPQ^EMc~-F;6^-aK_{^3Yso1*HTMsd7AW7OKbJKu?jV-P)ikyj!Qkbym1%oiW#Ws@o2-_ zm^qKlvw0f|-X=|}!5eEy+EX{^ev|TR;!+P>D|uYrxJuGeHpWBO#`sxR(k^e3lbuK& zDPCD?cssPyvV(2jt^jZ2AE?2bl#_AxMomwQmOclioa}*G^1HmzDP@r2(pvgMHFHvj z=F#)UDmk>_?Tk17aJ|jjFz`0K%hme#G)E6XcU_tJ{k*Z0Q1ZF92p!swp107<*6s{% z?|-GU^Y0gigSU3q@8a!bJ$u%KlYcfvo_NiE^t?sx&hU2l`Yx;N`)vex+w|Zr-V*ah zW~impl>U{OW-W6{{!)4S=){wbws{)~-d1&=koG;5cFoDeXy&I5%~{-?7SmPeE;N3Z zH;HgfHjl^LZ@*u3_avLQQQ&Rj>;$}V!qlGLYTo>1`itYOh+S~1)S6l@Z<-A_2-ZVR+hdQ)RuB8sm6Y4LAx0oHJT3ae_2mIfm*V?>Y3EsB!x+d*= zDxJGDJt;2bNfj@Lx6p1m4eHQ5^q0!prw?57Wt+D#;BD=VCGo}yG=gqS-ZYa5<**#y zDqWfPbn9N*53+e13*Kfu(LlU~PEvC^lBaojD`KZoPCswbDrs(>xZ&;9_dMqOd%$tv zZAia~?t7ZBhoJ8ZQq|SuiRVpI#dA7AlZnaW=Pf3)wbt-9Yvvx#@3$Ne-d>+ygE!`B zDN|+78XeD@rb;T0<`g7Psl5H<_@;Y~9!{)wK(ALEf~NpY!G%i@i@ z;7%n?e>uEWXgBv;hyU#KZ#HjNgST4;PIBK&=aFMF)5G6!?pBN(^EYK>7#T~ zhx($Lc^Z`r(E1KuV*TZ1=c{?fkUEux?5 zDHM@8RZcP%$}`vI<1NQ73~&3+dvLtX+eGj-VCZD`J(YGR#o3$2Lw|E=J#R(tC{=#` zsVH;0yz!$6=Wlpx^Pi8LYV$S;ylq`pgEvhN1@)Zd&SY8nI=WFoB+n7BtPEAj!W*+LI^HfhErlQm#b4psXR`M{e z$?x*URm?_BkHuBfoj^}T8{SUr|DOHr`|VorwtD!r?t99*QZ307>Svx9t*OiW)l&#v z#SD$6xfg1eH+D`Z<2kh9?Ui3Ve6G#g6!3PEP|@)iro4bE?f%%YVOVx=MBbXliLZQa|PJR;h08w>?`weYbtT%>Zv}UdhRuv^&jY zG@hdK=i^Q4N{VZCTC}HB-oAYOtUYbst^;o~#!hwL)8?8Ue`3i~D59Tyys@j6qRZi} zG7WRT9dggmciOyN58ei@sll7{l|!m?m-6!jH6HpywVyYsI^)!#)upwJQ-|iKzc}8k z4h?S`518E5=4~c;>-2gJ-q>v_O;)Kx^M|gL{C?g-yJ}>AalBc(HoR^7%)8pzyv+h{ zo35JXzNc26n%^gCJknWf)=Hk3oP4kI@x~0PI+S)K#ihDZyd2&t-5TB=_{ka0@8g;c z-j?4{gE#A*tjSxbNAgQ?PEVRV*HTM<=B%CqSMj5vniZv-Qk=CyJ*-R3oPORk6`76N zT5EW_aPuB_+V|TH;B8jx>Hho8%GtzTXk6-(d8p~vXs*?0mp87`RFpi@xoh%=dg!NS zZ+_l*V@nyBjoR`X-adH5+nwLvI|sa7JFyOL%-_;ANvNIzBh=DPHJ;G5F*&7LjQe?` zGqMxI+b{0=*mv#w?MCp{Ykf}MB71APC%NA;b@dc96`O0Psp`7CrOIqg&+zuC!=8ND z=4~!`d!gM7|9i@u?9;kSlkg+uWL;{hH^wDhJq1lgDT5T3I+SWj@zDJAGpEZNouQhU zxwf)v!`p4^?{fbCg?ZrZo@)bmWByProkD1y7(aW^-)}J$8)0o)-aI=pyuJLyU60xK z+kEi0;NFsW^YoeIZVb)8i#PuS__?W^t>LZfgdaG6p0xnHjqh}w{ykOJt;tW0{%ht5 z?Ln&RpLS@KM)sF_zqS7KSGw5u+d}ZxZF)((m3bm$jG8^9<&9OSS+Tivsk|L~=i3jl zd0PbDwmeXSH-1e>9Xc_t)o3Yy=m|=-n7?`o8YfqUYUbx!DIV(4OeUT;cI(k^y{ET) z{2rUP#o+DnF4z0tQ_YSzrBFZDQm4uz2kb*)@2R&Pa+&6Atw=Pt028p=Qo}yfFhaOFGmq<%x;s<1I($=IghQPwlwL=4}Od zyW`Osyv6K}YjgCRT!`s;_e*bL7+lD=MKlU5P9e>P}KC61pJi2n144Yd6 zH~xr48jR!#jfdu^-`Z}?tC9Usr^St#*&*Z9^hjFrOK~ZWiyJdgGXvw)^jO;9Hu3O% zewg95Ta~vHGT!<jle#y#~k{&-=> z_Y0?HymeSohquTcn*DnXes4vck2hur)tn^PGA?P2pB`$LH>rvg*YH`XXn4E-8`lhS z?zhu2-d=vT4sYz7TI!8)`Zbzsxkt@fdYDJj`FNAEHCJ3SrFh{_!&LrKc{}^Tf5U&*w&OiLBjfFs0Wo-%dJ ziR9xgrec|EIVC@DbjEa8$z$%fPK$rF-FZ*X%y^snLJi)ew@HdiebP@&KjYN&#AxQE z&c_=wNL__`=%<$QGp_Obd6P0|;+Dto_V^|Hb#r(-E8}g#h#Sj)Ppv-dzQy!YbMlhr z=Pz~j6hgDHO6XepH9E8wb8@ZAo8;ttsF_WQS3HKd@&7kUM(4ZVG@0qq>`3Eb&QR_07MelnH@vkzee>rW-pN6b=ocK`&dqq+IC`%8o_ccDy=P+9YWADT(>V8=W=Ano_4B5wuJjn* zPJHo)KXQ0GFXL_TEj4&k=C8?K%w%HfhORA#x6rPN@R!P4>jUTg#^J41#@mEf8;G}} zS!MoRyd|H2x!-Pn?{Q~4yq%x%)^prE_dVrj8PACEGIbF>xYI78O3ai}^%H-;DLXN| zop8{aM;zWR$as5xOQCf3MnUEUhG+nChLtp5)VRx@lRZ5p?|hru=+2 zy#4H^r`?~wU6S$k+{DDZDSHb#9ZgP+ho_&9x6mrA*jzfEH)ZFBw|zSAf3tJHU7GQB z=lTHNLch+GPDjdf>7etPIHZ>L%RJQ5T6$dGxQd$98KeS3kUFp#9*6j;NeBa^ivW&M4?H4xAd#af#_ZX@bksf6|+TZ z!`lZIZn_4%ZG7wB*SE&+b67nk0JzXbL;W!uCQMQu_ES9uMkKAtELGIRHTg9jsg?^| zOg5=XC2oKl|Kmf(XTVi?YXjb{yRQy!1@+eSaC-VL)T6)PQcWOLp3sT8yoF{^`VDXG z&+2fZ&0AaWHn{Vm=6z75TSd9Y+)V0uE4q^+@^h+w-sp_%#PHVTiIzgf9l_g;Gi&hXxig#2nm?4GCL7E{4>dhpTeMEgn}!YUloQ}iH6HpU%{+SExQbe; zV#UqeS;_EogU&zp31MR%piAKIJCTWAJNR~3)pt;hHtSJ}K>4&FM> z3E<7*x#qp0?5%07CMP}2A2YY=DM*=Fg_?e9dZ;CjrgLen6qo9{yfFi{l!0-NHoT4B zvip5DZ(YFKtB(cn=Bd@hPfPK*_nU@+GIz&*s6%^jdDCRbQPJ@Bo}u5l*ygP(c)P#P z(#C&JbL_n?|1NKuQ)pz5G^KpJ<=Cm=ZSDyRr`x=B18@d{NwbZ5Z_QfMx#oG<4UYeom%LT@G(W??^N4QhD3=j`8>l<<&pW>IvSSURZ-S&G#(ygzC^! zVE)jx^fOQO6d0jX(na{0lbStf;`C6vyz!$_6?!DiwH0l?e!FC!;qSNaw_f1wsi$l3 z#ytECVqJP9&9#yzG%n?o;!=M1e$!+Koq$xGan_acGfu5p>+;6xF`AuJ@)+K}Gxn$> zY~Ff%a)1!kk>y%AF-bZx4-+>6VbrV2Aqdv;`ayXmRJU$%Mc3*N>(mw-3+z}~3o z;eI6#{i!tT>hCvpNX@R8nekMb`KdMeUEVaSB#)=M;qA&Uvo_eg^#gC+hpqTmeosBq zOMXjBCm0jg^q-bDPq&7*%Rm3*O*U`+!Q1mI{-t^IOs_@ors=;_-j@7)_O&)|1Hjuo zTWjzpebz{?iDuWK9@e6c$-}k&_f)DVO@^5@TFS3k8|ul&TaKNY`|Xqm-+0O9Z6J7C zGIC{c@2NCZ#;G|;DK2>=zZBQ=CS}mX>5;T%M?8Hgj}*_x8@r-r6)SFd8+dx(*KOVg zfwzS>7swm?X*BKUt;}x8QcPVbPg>rj4lC6SZek2d$^jD`XRiRVzFit-!N*>0k zxt3a!pML6m_gjvNhPM}fzvc5bZ$rV`tG5R5#>r?jcP;fVd6=IbYAGkzQgdy-&u%Fj zGt)y&PpH=9;WX&?^Tt)|ig9N1Xv5o>9$(qh=Isjb_SE(O-q=s5X8)mU>2HN!qQp40 zrdB@QSf!bV@=r0Ovf z2+gVIEvBo=TEpAVK0V3#eGbFH+qAU-yhZj?C|9AqmBU+TS4H?s|Ht#cwePpl;O+LdtNr(zvQ|TVZKmltp0_fY(d-xq#o^L8b8n>D$G@uu8+UH;>Flg>9L9=bb?Kb|+`jtp;m9eWP`gPH2T zml*@zhTYS`cvJ4Z1l}}f9olUXIZNg3prv0r%D&&mg15FEZz<+tT~%@Wy_W-xt??%h+d`n*Eo<8>h#ehQ`^sq?w=EUAyerx=Ds_9?j;a)U4CTBk0LMw*)HM^!q^Oc6n8#5%-hPN^Q z=zX`%+tuK$*L8Jxfw(H($l3x?2hdsp1s(K2{Os&bR@rQc2JIz{+KOb*WW_CqS zWnypLZ$sX9ug%*8@b>1$I=nGIPr<4c#G90pacb5L)l#4S`;FD9L#uNwwUmb*>KK1M z-dLSlnrFphc>C%;t-oaRb`5yj+U>UZ_f(ojXk6M$XdaE<&zmMg(VkS($j4ib-Mz(I zzaID7yiEje_spunTdJLg_8e0;bZz7F7CJr2@A4KpD=D)TH@t1^wE5>YZPx83D6*13BPQ%-pA3Z(S=4~=~8_{cx`=0WD2cYIV z&Yq<>J=CFou9b9XPCswVMlDq@GcNTb#r3>N8Km7=e#6`E7WSTF^L8zG>wjYn-lY6H zQ{2y+W~b79qFGy}N6%a2jtp*#a3-gESa(Juk z+VJ+=`;Ojh^EMT{ZRmTu`<{mGNaK$=1v?16q4TV)HN36-$xps) z^EM5Z&J@vocj&cp*daNLNkPRDEW)7n~yiCTTk5Z*6*vWCfU4A2XEsxm&BX( zwPo^G!!w7Pwes;Mu_47nyDGx3=Pl-pDr*gI7fl^^i_O~%@Ya3c+H&60A|@G=)8$Q* zP2!4iYE90fJ$l|EJ2AY?-R)1#&qJ;QZ_h0%khhr67isUzNiF%Mcs|}(k(yPcI6YSutOjqqOR2dd#;NI%H2u`{Gfqv9 zMsqE-%Nu#1lT{d(H2u_)pYc?F=JE3;Rc8izsOhO_bHDv?-d9H0yv+u06P~ZZo0X^L z_jfTKsb`)z^V3gFPpGb*f>cqHEwqzRzorLzn4kOd^TrI3+VD2;q;ETa|Lq3w)_?e& z&HJ8mGNJp8S?k_!tS)7i;#?c5B~BO*%}IY+-aK8s_5H%^wjJ#IZ4P*QWn~TCikK=V z&Y+KM{8C)%LC+gAOIoT@*`49-#|M1d`TI!o!P~S~YVaoIY$WP>hBL(0Nw_VU8jFfLz$xq#a2Ub9vKb zpodzjQi&Vhc0cBOBQxNtye$N8UDwp%O{yiurGDtA*4#y*Ihmhp8FzW(MCE#t{9SL}oSZZVr zlTW*0eC7z#;x9xpH680;&=<~r06LX#~VAZ zXv5o}e?ROAo3~})ZREs~c(Zy-&YP59I{o5!lRDv7+LEV;c&WT~{?#X(pZ6{YZ=KdR zC~p!csZJ!Gw<0<$B4=9OJRO?*ZNteU-e%u#E5O^PcK5jNDRYK?R&s4BHl#eEz0vRI zjm}innWu>SQcjmQo{_XlihFipcw6?Bn|j*3tpsm(Ut5DW=4>>rnS-Y1>M3ZlNvFd& zwN#z8s3pG?m)1&gmp5jXR?)-VQFCoY8{RJM`0Sk-ZyWa5{n&$#JN}p{eOC3Hd35D# zsHOMTz>O(cc4yN5>#fY4#!OqY){k4vPOa(&x7YUj{qHl}cB}GsLdM(Go$hs?)0iI0 zTx;z&=v$D+;Vsp~q-pqhlUC*E%J8=O+sA*x;qAnXx9-zhFmIfR<`>!0@8>PyDKvc+ zFef#;N~M|K&s(aBl{|*GMKgYfUvIhNemg1S?b!z!ls8U{Gm+x-XtbX7XM?zfXO-fr!3Ut_+f(!My!7#+`B(H+L@j%!`s_|b&g@V4UO z6My9Jc1p(EjO*+0W_=6N^g6`!$(%ej#$)np*2eRusZ!>S*bnRKdE+XLX2ps&ynXDR z&*Gmi?08R4&3GI8P#xZ^oVj@;SE+KwOxZnStO1n_@P?OJ8Ihlu=y-D$Wyh-1cNLAQLb7?%`M*>!kpt|@u;Rg>Y+KJ)PwTBQ;Ee%>@4D$m03HtvG2u5<3UGcw+8ezX~QQ}*A) zRj6Mx!^YvQh~04l@w_pcM;qS${j$HE=J0lA#@m$M54i7X(G&3W+{9SXp2p#=oSl-o zA}UsxF}&^dAGecw;}(_bHNxta2^=8qKvD&76{-9zAc&#te*8OCH80 zE#=`7fKyq%r#*8RyEyfF_o>lUH8_t5;IeY%rp zMNKw(sF^2J$K;XL`gxN&l(I2SUCCp3>$CQA6CB>o$#{FS-v;+Rm3r8j;(p#pK6Oi+ zKtA3yJGCZbczb#CkzaRsJ2&I);ROM_k!Pz`)7M?}u;)-e*M@4T7W4Rdqcc=%CLnpF zcxVsw7snf`RJ7r3)E^%@$>Ht1jJMTK1@LC&Z(^(^#*5>v%xUO9vq)V*9U9(#|AWsz z;_%ig<88*Ejl15{or67zB9d0u|`<9go8?5c=b ze%=yRG`xL$%uP2qyj`5})?>s&@$aeBbIj)pbEe9}oW=20M7QibRJ*(t)S=<6`{)^e zb9lQX;9J0*3JGc;~>Zg_ij#^r~AH}m^c4?eCnexJjLTXF(d#2@vAZaBuT2QGA{ zp?>ZnW-aqj6E*#=nSq)eQdc|%xIe7y)I9^P%3B-o*8SC-ys?Kewe?9Abg8k=_{lR{ zBkQ!hd3I`e``X@Z7TdhF1#d5meZ+lGQ=O@&x4N7Z-9zK^CQU$!OVjrA7Me|}Qi&Vh z9vgS+P@A`Q;O(w8HF)#ntjP)QRchv8oSL4ZHS4;(@uTd7@fa=TWV{IN=PhAH!&|2} zw!YKmtvz_V>Gc}Cg-%t|vy?M*t;R2T*h4NGgH&gxTKkrYh$#_n^aw@!mg<4 zujG8|e*5U}ZnAmn0Nxf{{iyq%GSAMcUEZXfN)x7EUOcFQjx{)x?77w~pp`^WRWr&g_+KPPEqkIS2MUX8GenBM%ng=Y5n z4Q~e>fAeiNZ(YILf+-Een`bY{d3JeA#Z{TR(Ddx*O_M?CF}!U!`TA>Z-nxOearZS4 zZ_0foVXrveq+O6hYRR7}UL0@SMMWFlzI%56^K9O_gSRf7pV+zIQ_h4ZSUzv8UZ(c* zCe0^DhlaP!FCEj%=B)>Kdvj~c0Y-k4FBKKAIG^NS2Zrb}zQ<2rn)cJVJu_MFV`K|xt{CQR{@OE9-C-v{C zrU!bcH4~$<#!W{^6u;)b_7zR+`To40=8ZNUxAz?;=)P2P(3#Pe2khpFSTpd4 z_Wd>pysdvSCvS~5)zE&brw}vY(2AN`8c%3lJ#V3zEx+My-G|Tnt7uMk|rnjPh!pZ!heO6C_ z(_=P`=Gs)6bu~FPewR09)?}c^GiSrwfB`-4ws{*0-g-P;hd1^{%^ny}rI}x&S(kn( zUOffL$yJ=Nfzd zTZ+f!t)R}|dQbN{slCnHF!1*5l4jlsQEQvIgB;e?l{R(c{}-(-JGxAhJ&{| zw&dik=t&lxzZ~9p?x}R>WJ=}jp)daBkM{jG0=&(-;u-&Y%J==!LFYBy6`{N4w3$=V ztVPW{FPS4sV=l%mhN$a$@X( zacX)rwOrnqP0BzIwd9xLQn$=W9h!%JKW|be%%IeUxAlMC_<5VRQQ&RJ);hdNwG#H( zbZ1Jniio?sNmXKYs#%-LBlTY@Z+qW%)C)Fmqruzu5zqSHQ)w4P$2F%A(}SM3m@191 z*3TO|jGVc--@1MEX6N?}UJ2eFT-D6H#q5zU4mD>eX+3YuOwG)UQ`6I2TF+Z#hlaNw zoY7^IeZP$XZ>wLf!CMRikv%7QTQ&Xbd256n#q>}rZy!AL$O~-V#)7wLS8j=aPo>Xj zDIPPuRBPjTBh{fh)%fY54xNC@8?#Y|W~RTA$J}rGKX>Llo40Y`ZS<`Hym7*;)e?2} z6dGYiQV&u*blNU&p&68Z!`s+>+FfV!HXgin*j|S>@-Jy+KTU6?G)d-{a%w!%TE?rV zz-$`L43eK7Npk|sBl#H*^}D=1sDNJjW^2WwPj;@w_RgVR)N$^p=tK{WclAJv^xfZ~T5CMr$T1?L|{p z%CA{lJq0?Mjd6|Etd%^BOPW)X`r%r7T;BLmX%*v=W}b>Ryq)ph_inLyyB54Xa(4~h zr2JBx(~vxk7O$Q{qjgn8KQ3>YDm+ta&sxLVpohlpZ}T<93HW)_>_{TqT5EWF_>wD*wRxKc-bUPC5^vV$LeqW7IJLExCckp;O@8KKem`$? zN*ScMv{vKiT1hjHq&X#*H)fVH&_gZxD{;fyKi)R`8k@K2;H}@~FZ$n8PyU)`+Gu`G zo^$i_#$9j~H9d`>rQTfL3hLJIcEpvt9clA61H8RCqnUXl57en+xh#J*MYZ(Ha0VFoE|?zsU;8Np*kOLny%=Pw5HosYxD7zqq@1@dUtyN z!S?-jJ$PH!?WKJ0Df{8{7^kL3()3en{Pb`wZve*g@g`N##8Y{=3u@-)uG8|yiqw_r zhPRH>{`f_kx0&GW)>%1u@1wW@_b5&HeVb2~)ml-)}d7x7X$*<}F8`HThKTG56Y*%UkGV^t^>;w)}>-N8kR{ zAK1Lj0dEgIo|reQhnhST!DML8=H)H4Tg_A@Pg>qQog3cH`DT~*+q~Tf-j?>=mi9g6 z_bk+&K5H^5O^-Px595;Gjo6%%w;aE(sfj(VrIt=s!;iF< z`Kevr_)$$o`l+Q(m{a4I@<=ZQ^K-4s8>>sJBoE`%m5SzmYk%X}@3MKD58kdC_)6OM zlsj!??ef+L-AXLQ^ycywlg+c%@V02r5B0ZsTL9jAEDqo;bQ+<4&t97D##EkNytxyI zJTJrB7e4v5581pe1aHqh6Tq8v){%WCXG!Xlad+DM=+30`-ETS0)bKWL>K;RE-WGwk zbwghL*LhDlLzg#xv=Mld`U#Ea<1NPo3~#-TxpcYB+hXuGdszbB_%)|SYxcrC8b3YK ziO}Qn#*cBr?QG=ru}#bbE;!`Rs!Y~GfEw{|xr z=8c~VsrJR3%Dp#nE3Hk-TdJLEb|>vfTI=#AIT>da*5Qq{ zsM%Xf)MK<))`|;LwpS5{g3Eoz$E{Qi!pG|LH(H`Ee* zF}Qu>BUk^w47c5?yq%Eo)@$tR?sF<#tDFWs?1AxATFTEjHS0?8>M2N0u9CF0R+!q?Zk|?-fL>`##+s#d1^)U5P!cF(Nz&SUEYeQ zQpss}`{VRe4|RAuDdX+s*P5BPBDj)rGTu16v8y6T(DN4AiQ#SVhn9WJ;qBy%x4W-; z!+lSS*mWc1MCuExscGZ=s(T^ixZF zDI%`tt+_f(wd-$&||5 zBW(|G@3+%3-nvd~2HvDoi{QRxuu0k!#D;e%?ZN;qe>ZuAV!5Plva>x@$o0B(k zPOX__s6RBPCVxC{>`JOA#kp3}(ptu;r96J#q*XcMhPQ)HJ!_T2+u0d!V>-3bzo$8R zDD)2We6H24o0d1tWMaD2DjHqs#PBx1)iVPf-pbua$d&UtzFu-toKwpS$-u$EpeNcH>rvyuIX0ek!tCAi>XqnZg_j}lmBz8bHBC9 zc$;&53*$|iYG}M1-lWft&~9V=dfsBHRMr~a{`cPozJUe*%kNv;!Y3F{sAmgojk9N)bo+|fJ_xnw0R#Lor z3e8oWT@~Gf%NwgF)aHKs%2n^53w@;nw9% zvr01oE05vrZMO_>Q)yq^3$^4AjW;iEnjL98nq9lRX;$U%7~Tf{ z@*^iWyj`5}Hg#?d-ZZy#j((DV4JP%V>EGpz=S$5hn#`#@tSf1$Psa1{mSY!&x1VkL z>PUyTOETWBc(Mj>((a0gb27{?#WfzyT9-FwZUn8_g{IGZyyd8Fczf`PcaL{?yENnN z^?n_Sdrx!pT=NbrB2VMkn5syj_;@_P~MydE+#AKTBHL zi{xkiP+dI*W)96D`HRRIT8lm8<4vmQi5uQ_8-47X;BDhu|GvI8exJiFPt^d9?RpB- zyo^Hq5@eb@p=-IXaxOSl)a-~JuBE0&qtgQC>BIoHdV2fL8E{qJ+JLtk26Zg%LDfuy zo=|?cHdL3xTWD8BOdu_9o*fz9{@L%gqio*Vg13=N67W{Ul(?^${L$eWzZO!r;yoK(_(?fDjQq{`GThWtAm7jU?@x~0wnH%1Q4w~5`1Fp(j zNANc7g&Mpm^B2lej{Vl`N6O>pO_~6+QA_?ti2HftE~s;KZg{(}`=`3tymbO^{YG>y z=RJ+wOHGzSXTly_-uThb%#G~#^Coq!nTF*tyzM@2@)n!7&fx8(n*w;V@+VRHd5R7HxjqQ*mi z5!&TV)0M_!b!&Kg=qX*JxFn@pPGDzuH{b&s5Q0d(euVtskF3f zX-AS@(~rxWW)&+^d)6A>di3f3n9W;P@HXO>0Ny;clCTt-KgQp@yv1}bt(Bd!;EgSSrOy6E3i){W6tpEd8snEaZxdfqhI zVs^yOK25DqkIP$3x1P0zw;}ud@d*2V>jB=LTNA)r(US?CB>kbiRZk&wRj8jGYIG6# zUEVaEX2`>iKF=H zb*Itzg{sx}v=4ae(572C_Zu0YZmyHs#hW{U$a67Yzg>3hqFe0ytuJ`nG_ekEp6^GR zzOU4HLb;Om&RWz`etM|gX-gRzInEtPeU^H^jehf{&)K~718?is*Wpdtg~Ws8*TgwJ zYRy`WhdEu|_)%tKJZ9H1YuS%Rv*&VntL)C)Z{NGK|9YFZ{@`s<`|jnurscJEgwCjAlN!eJ@(uTKAKm7CYHg5yK+t?{Jc#~@Jvxc=8r=~~J^h-L`TIToj zMklLFaZW7ML%)=V9`++?`t`hVl}59A#cz1qbN+7o+q?|~ZzJxj!5j0GsX6VU>-u?< zs+ZZJ)I(_8<&C?b4$W-&4R8N``k!XnybS_x?K<};=RHleFDZ`{w`S7xJx)H}q->0b zcEYu(G<$G)OI5|nV|Z)*hk>8Bc^eGg9-UqiZ&q(j=g!GU9w{EPwt5ON8Mv03-AZwK ziq?AGG~IeUhPVIz#P4R=ybS?wOExq!Z$(dA%3s|5CRJ>%xSqGr4lVW!Z%6&K^)WVY zL&4jGu06|nPeb=2`K>)R-IJ1EitpkregduVU$0f>Y<=5<1t#w85+-bzp;u^8{YQ+^xN{WcuDJ=3GtuJ_bskLQ(R^0{C6 zB5LV*i`=#0?Va!a`m;7~Bf#6*8(IKwk#ned`YvNdoOL7Q)bkeEt>Nvg1AdJEu&w%@ zjs$PBAFIQg^ma?V(Jy(VIM-5Z{E|nCr{#^^YP8f<5%E%aJLj)w54Z2PQQ+;GKE3tt zY0;CE@)vgsn#`K6=+Wei@kq58=f7%kd80E_vvaPk#LfNo=J6N&ht1n)@HTc{0B@|- z(zX76E8;aTA}3FT`CZ=lQLc*AhPSq(hRn8kyAr&0-jskhi}RXaFVREIJ#r3`hkj~} zpB_DLT*Zpi8js}XT59^K>7kZ9E^l0=(Nbn>t>Nw0cdh!8&D$98)_y=A{d>x~EnVw* z3*9L{Yq*wiYI=&$E^nb#l-(NMMhyNQ{zIec_X}ge+v5xC@Wxuy>{*J_(`dSS3R34% zywP?c^~3(-d6TMGal_kHPd%`=eZP$ZZ!4Y-;LXZk^W6;fa5AB5WBkS4Z=qdD6OiJX z{9IcqZ~yjM_c1naKVjc*6TsVjSM)3HJxw(+%}IsUs-8mVDorQ!6rG>7BwZ?R zpa0ts=g;e}0dEUdB;c*gsq$w&(u5@s`*iO&X;o+y=3zy~nNy0>BjwSo_4B65Aa%lc zMVt5Zwd;P<-@e}_g11Rq6Yyr$P4dnbozvy5h>GN-i2RzIe%_=Ep`BO!hPUev?el`o z+a&PTXJmi>drJPP*{9SS{nV15Iq9dSN29rx+U1QO)nslokJP`*o8*k#k>PE|ci(x9 z&D&(~*7oK)yh%NzVupLCmg+Jd>gU?(DKLYyBgsQQJ*?6Q+U1R1Nu5h^uC=t`ZNSeL zIRAd(TJW~z<(#}(wOYg*%AVc(jUOdTjB9kv+En@3gUcH~s;Q{-7~byw*oa>C{Wb-> zt-W%9|2c1w#?17p+NO5}n z_Zyv|T2nnVXN=$FEmeoiQ_+UElQ+Mz(dKO$cpI|425;OQ-*M)rhnjxIB~3qdjGuY< zHxC)Fo&qDR$W_$za4j`G)S(`YpZTe|*5!>(?vAVIQEJ26=#xLu*5++GcpE%^;I8+y zxVuhyQZ;Y#RBQdbrK(cNV|e?^kjLL)^ELy#wOLz(H|ZXe;t6}MIr$>$?%a&=MD)Cc z&YAv-HoQ%ra=>FYZ`Xmhhu^5d8@p|Zn(qQN_fkCtepJ$2C23ARCQr;-_P{<}-dKgs zP|Z9QZFoCz)0ZBydAlCG-7#TM`R^(F4BcZYzy5xcx|Pm1Rfnna`+1XgC+))HH@vOi z=YV5v-e!Wg`FGXeE!9q?o;C5Jd(iWy>9)~4oSw^DLEReO+U+y)cQ$Xcz}uB=2bX`p z6|~=)U;9b5c5dFd>$JR)WLBwY!`rHNcWq_!HXFQMJ~=0Ep1MKjZuMN++O)iBrlIkK zPD7J3EpMI<4Q~ssc-;B-GB<#?r|!{g14dfw_x6Q;+h%KQ#}P{Xf(}k*&E~3^iUVan^oQLcI??7bl%f>;B9;7 zq4Do2eRhki-37B&4+oF11quA=5DNqhW;x6kx`>?r$w zn-AWepHYW5i{V1Q7bvWa)r)1W7`u9TDUc;1*z>agN5yq(#5x4mrM7J;|Pv+D51 z{4tYZo-(!6v&)<0WVad*{f(gGc~kDf@b-l-KkR&ex){9mc_e^0W!8V~){`FP7w(eSqVsdqbne!2v_ZRt78|DLigHFp%MQ{@S* zOMmqg8le+*RfNvRTWBYhNgCdc+`Q&&`#oI>-fo{$hqutWjqE2~`unYscckg99NsFO z7~U@Z%ap!0Z_B{jtj7a*<9TUzN>9vM=1HYn7H=`rb9sx&=9$0Y?V@c5IsgCZa`1LV z-{JoE)RVvI>$H~Wae3n@P`AY0v1gYzel(#rygmK%S(EMiZ3TF1JHJ5QLQhTO*LXBD zba~UPq9;`+%&F0uy7c7ZjTzXL6t8#;Z?7J=rk~B*O7Qm3rUH3mpN*h7D?M*qRpu^Y z>T+#9-g4~J@b;Hy?;4o#wqcLmkNwth#~(AL&#Im?kFLCjO9zZ7?m5-`n2_5F^-F7+ zhnoKCA@EovCs%1aF@CO%(d^TY8=cggjItAh+fQeI;+Gk2yH$BRA>-}(MFsMv?6dBd zfuVcX`1RKtccICg%EP)b+RvLNvt~z?JchUX-hKFXhqn_m-UdF?0(jHRB-E3Sx6mq$ z;E(4`xeLSF^f9yW&jEMbZzpBEy*YTK`<^O!u6r*j`>dOX_k60H)l&%FDLbJ~)m3O7 zsRzdWywMq2o&HK5!&}EIhA(jLx05s8o?2RiH|CLaXrJ^`^TZjaW)F;0)1%SVQ(y*3 zOWEkBhjB^MPaWfD9%`32R!KO2!`t`H9CNtC+bJ1uPdt~CH}0q<>gp-9#Lh!|c6rle zAcq=%#bbE;^rlviI=r2l@pkL5QSN)H%wP8%OqD0c-s|>OJ%u7Nr`mN)9_}k2Z=uyI z6%B7UFMr@3hqu!*-lnf?P~JE(>d-kz{>HiAG`rJy{5vIek~1=Hc-uCt2mX4h;*?&-NMCoyZ;kACv3$PA1(m)7)?k2mf}Q>EfJyv_gmhc`Lz>6saC_uZU;H}+P9 zR_-^5)#@q4WR6)|#FV6*@w`cyt+?TBRO=%Ta(Fu{<88sV7QmaeuOP0ZwZ+|UjmBZA zyj{9`w=E8DXJ@>P8*^p+dn%o2syI)OTFS5S#NTgH_2!C8yDpWtLq0wBC5N|jGTu7h z)&h7d;so>EZ>c&V8`P=tGf%0!Ju&P(UvqdnH{I=r2i@pk+8G5+_oktZ428~tf{)0}%M4|k!-spl;;v*kD6FMM+9Wc>4m z9X}6gmGL%XZ4KU-hkw>4?T&tG$XEHN5?O!>}_Q-Y(2|+jv(E-gwrY z{B_Tic^XaUB@?ULla_+YqO1zC7?ZKPxxhM1%P)khRS3 z-fyuovr0i5-kx3520w0a*Zp>5iMI~RJ$SS7S7e5t7tLCYpB_oG7Cnq7f4b?^XsHV0 z)Xd4ar0J)oUy8fD(Mip0)bvnW9>d#F&pmynbHCkG;;nUyq3Pb!kk3fSTJASQr@P^xR2J#W%Hm7N&gKD_2r!yVpkF7dWuY)amw-hAWAy;QtQeej!RN&3JGaJsGacX664S9V1QY|Si<#%?L-NGsEuj;`+gJXz z>=WQ^<-7mBz7c+&!@OIErF&4N(bP*kAK+?X8gYS3*pUJ5q;WTVwgg;}x5nUY#KbbZ z`94kwJ*2+Kvd+1Ox9AfvyiNU|DPOR8YXaUnt}4SDFHLGPz&JHM8ZE7r;+mWqPjT8( z6)Dcl(x*~dE5(^p)21#3f z^PV>U(%QXk-kO8ARjVuTCe;dwTYZ+j4SYSmCl!9DNgeujqVb2+O6JY?91}V*ynSuP zIrrGSwE%C6JB+B`_f&II(mca?lQLH;E=?nxH)f{hP786v+k*!Vnr!pd61hdP-)OQyee>iWNDg}?>?fUs2!e6*7{(eX+@YZo%4e_Sgm*%82p6c?Z z*=a0KT;399Yj}I}Z#V2~-*2~qx2>H=*6({7aw4_KTf%9kzTsWovZzux3&Yz{@BQpL zo43~B?WtK6c+04_)YFUAPi^uR@(GgGy1Yrwz}Xt!KK1oex7ob40dKR{SKv+Rt={9i zA8!&ZE^m@E@JtPFhqwA5{=swc`?t2>ZOHAT(!Hk{^_=?N^PN>X-s*AJE^ofGQu+;V z-TQp$bo+j52j1GwOUawEhurvKKN>3ZylJwrdadbPc^mNECR1(RZUb+xY%If@X2Kc| zCr>{$J$Y(9Z+UjbTAUI!JsKUCw}ej2{WkZ@%dG!z_Xp>;2X7DeyffWjIS zSUGE-H_e1Kp18avR5ZL@cf!R-+4oxq@HTlthP);8*1#M8e1uxFR`O_=ES`eKsqsji zR4dL&=E~dL_8b3Z^VSi(^?9|*yjAN&vb*1Ejn7{O&7Cq>B3#@K<`*828=h*jKZ}8UQ(LKCn`1^0XmjX}I z>pNK1ipyKVjtp!Yw%RBe*mLYFEcS$FlU~0x#D=u%$Y-z*WkmJAe zL;HT~1Ku7PIWFCMn)Q^dy+oNjbJos#n)M{Ja@V4KAUmb4q))%}@c|0fHtlqr% z30X_Vn5TAlV~3Jv=aNT?XUCgW(eU=y?fcHMdFuz>dNv)O?tZgsdGS*VYhB(#&NO64 zS=UYGEu^}&*6?=lTa8Y&dFv0}c8rd|oAm2w=@hI!%d)ERX!1+FxxCRy%_@wuL&haP z$(^sMnH_f?ipO@4Y@-tw%@PC~S% zXP38-iV15CZ_76~9%1t~5WL;jVnThrrv!0^4FYdt#?={b(u<>+4?Qk#{HkUwNzLkvOIq?X?yIF*j58h zgKXZ0fVWPqC)U?{%DsnZm$#5?JhhNKTpN;;Ym<3n#XxO%`{?Zc57@j71#fRmtTJz$ zjAp;Se&*z7GaYZN&TOnwYue?FyP#(0Qk))38{P)&-0P1vZ^OXb@>Nyl&FZ<~`+3H* zIN5Z(v8xd6dzzd8HG7b9y1a3fua>G5@)+Ju-2TO%+Pn=1Z*$sBs_uKr=}DS1p@(rv z(@$MI1xA>GQ>DjOGpD3AwYZi&FiuU6%NxJSRg%{D3m(JUssC@WxNG9~}_ zG@<8;(~GsXy1bDYNo#h>wUR$q-Yz@pqkpyE(=p(!-|RiSRr&9eOPs`t=gQkf-T#FD z;JNtk7si6OmM@p#jc+tHC&0L*>8GY&6Q@VgzB$v~Z&HXUg-*^?o;Ewry1?V>z&oVGMe+s!ow+P@@sY^<#BnFoPlw}+reM>@%}b% z#7WX~PdU~<)>v{977{aITTFK9NuDtzyXA9@gPbY!5 z+ZU$fjXTnuh~$^zQchpLl*i>w$}GjD%vr}Z{m>JaH)ghWYQBFva?^W0VBc?(!Q1Pv zR^W}_*-Dx{NFJ-Vir<4Vk7h0Xl4hP{-k5>AV+MLOnrnTvlwXr4E^i4H4R7ze`_uz% z-ll-J^?j!6-&1Mw2|ZNgO0zarUFIpC0wbDE>d})cZ+$L)r@77BRPc8Hy&k;XjGxvh zed6e$mi$?tAnUSL@f4&E>7>qh zlpe#|uv1Q)Yx6b(yiLBp3~$W8yJ|gez7z0e(l@`aU(cIwX3KAQYjMmb=jZp{4c_k9 zmLYGPJhj!Q*P9!%mOW(0TS$kRwZ1>|JXhY@KRaooeZS2FZ#xIy?Y^hnHSbu)B^}}q zS<8MjdGx$V*`zo-ku=v*)6Y)_J&e;&O~1<2ZuWj|b zNfk9a(s-o&zHzCR%Nw24z7;LM;q6yHdtYHA6BT%454)3g@3)XoddS+W zr|R;SQANYs)Bjk0mwmr20B?^ro1OfgX4G4h`z@q5mp5jsWq#PFM(4v@VPb~2*XDfT zIGeYH;BDHN3cO`8`7Cm3CeOObylJx4izhB`oS`)V!&|2r7an5swg|lSdAb5`R?e#5 zO9}h+>MfbKSXF9adU1J6*pcDw6Hk7-yUp8T@Yb}|9QQrt_lnd~EqbVZ{nA<~&iwRH zOP=B>&`HfzzM6TcHF@Z#4q2OyH&&-+Csy3>)@^C8-L(Necu zTksg%hIK#d&Ju2W6?r?q#M>(?%J3%T-<{%m-lSbiB!~EYb4s-scX{JiBWlCjalhZ* z!Qt(K5^u}e%ypmBkX?tYjkx!QdsxaD&YP53nt&8%9*u{7jn0*~Bffj|0Ef2=OT0~; zT!uHzF6!Nr9dDdW7W^>3M(4`gy;C>h$3^dYPcJI**6;ZYdE@&|Eu8{AAzC^G<}989 zBN{DbW?afck7fe&Yc%W9@A4)&xr&;8Nf-R)J?*sggn`ce_OB9eJ=@Q7-&59#rKJhS z%I}-Uz29n~!z_AmdGno>vMa;ej<=2->hN}PiMQ?3%J4>peS51Hzsp;_bne@y%Uee0 zYj`_*-(wGUc)O&;+u9e(@aDUtTJS4BgAKoq_;2RT_q>#|HM~t;^Oa9Kyj@!2?Y>U) z^LbBw_ej5TzYQ#Lt)y$8w^+M&?>A{CA@M@DhPS2r^cmyuc3FwHF*D2Xrr9a)WcnFr z|BO@9L#^@9FKH=H@f6rOGf1l>kH*hh)S9)d>#M_gQ+8;0`|YZeS3A62UgE9KOJ#WT z-I1@K)0Wn1JknZCy!P)m%`P;aaNd+>VR-ve=MUq*ci#2;Ay<@mYtrqW?A}x5=fdk9 z)J$08DV_p5q?R&hJY1_hsZ1vj&Kq}>pbc-Yy?Xlhocry{5^o#kl;N!wrj)R+D5n#C zztzi>a^-FEs$;+9@OD*+w?{UV;VtX4mhx*(P~(yEGhRG}SlO7l*0i3tz-gHK?WE5% z`n$v1)g|60_gIkKd&Qi~1w&MwdC#j54< zR?E8!>CNRWBwND#&HZ-ocQ&nbc)PB|+jG4ay6vu zat6i?ZznG5{1=C}f0uY$xTp+oteXWJnw%OB`}EZ=Z@w8q{8{ggz2(Z=>~o$U?eO-W z5^p14tH2vO-<@=IdCPN>x$<`MZw@=p;qCeoZ=L%sa^KTDr@>lTf9jdPetF|AI6X<{ z%G<_=elpYH?S>L>Z!D=H-lU27PFTvrc>VImU1)S%-V)BjeE)XMBk#q3FSF~_Ak+gArHJJsRs<`QrHA1K3{m8a|^*$*|>N}7IZ`WeqcyS&A! z?wd#B*LXN3`ono+<^*kcyW%UK{kX&1EhXM|Y%9ZCLQXICn1>T!T+))CacU`#6i?hy~z)%G$twT`&V`%Zn*4+ zLF&^5&bJe3r&2{tekmu{`sU9SxHq4D>){e`Mcx{NxA_lO;Eg-27p*ywaNeY8XsUCq zhB5l7xt5xqTzQ-QpV#-Xd20gR?%d(Q8z(94j{pA`J=FA2b4t`)%eY2MYxTTI88mUp zBX#cUm+~+k&YRSg6*mv+9cPShWAoM&ybT$#B>6pM=hUo4t?_95QXbzpYf%?ZLGuaX zDr$B@O;3nsPDwMr~-LW^9w~RV5yj|D#gFm-E6e@W%JewyiIJrG~Ij3?;EK# z6OcU2LoNAzXHWs%c&GA?hv8I*p*+xz=m`97Ps*5IwvGakIv!t|7T zugJX1Tdd4k(bm9O9B%{N09{uHNNaWXD%>^x#c!`tTb ze*O)cw|3xd=E@r4P1?H@=iGd?%Uc#ze7nlRpDSj2(XKc5qCA^XZ=Z9cqZ(K%09lRrD&tm=lh zy?eDf-{!3&cw5r`!Q}Upzt>02T2fr&k=APBAs#(%QWfdk>DOq@l;~kyO@5a*u3~1! zsp*k)!Ebmw@ZST6+q`uGZ;Pf^;7!UO64$&4oGNQcabJ&~H)f{RbV5IM$Xe!+H1oT> z@vBmX6mi4bys!N7KQ?cj!P}6vRp2efUK+lCi#4nKd5hJ#@AO>Wd^0HhhPV5UeNQ)= zw=UqVP3MQ)_q1A*QSPgOJzqcfRXhboe6wZY&!V4X-Xc~sy#0RP5l7m*bp>x5W|rYi znk3(J_Q|-kRuh--Er5oprzhi-m2BD)M2cC za^>xxf4%7eo420eZA*_w^zW(lUYEV2=;6eqI6c0aC&m0yocW8VKqobw)EW={u{3w5 z$w_}Y-k5UQYR*c3zj0M9?^^29Ura;TGJ9ajOW8!p`zjK?ftsXwRyV(ybZbIQT=-=O`G?s zsUMd&O$Ny$#S7I9Z=L>p{SKSAKH%++#SwUu zcB+Z2o45Yp zZRveE@#Z_Fto`YD3;A?}tmQO(^W@6gw(adY+q?|`Z)4vq!y9)*&0XiIWA(s1dfvDy z&t1o=l^t)^WX$((Ext71vo>!7!CR+6%kp_oeWz#brQ-KvzWMdM`DXU@=d)8%#||S_ zH@r3d@qxo^-UflUHy$X%o2Cbihx@8E9nM><2`~?Jtcj7dbh}n|WqAACT}Ljmc^eGg zo_xCuZ?Se-%X!jGUfK~)gj(|F84u@8*{$Ji%D0y8Q#dPRF@tNGxSS28Ta*bEwz-# z6sISdH_h2G8!KAc@OIs`Yw;ga7XSZ+;oz<1$S2~yr|eCm zefO)$sqwhHX;#JZ#M-qcPcm;IJL13VEaWk~J@xSc``Y*02=MmOvMTcyvR}DaM0Z+C*X4$C9(W_{lnoeiEyEUo7)R>f-NiOXBUE(~w?ePF-8*}RPgZ`+zL*T1I; zwY(Th_>5G{6Km}r-l9*y@U~{dmV0g9?gDSmj;+93$oa!ZF=I+;uV{>ap0}>#0b18`>ClHr}lZ{ zufjqm?D7^Xv(=U1ZN)#=e#GW&Ja~I`Wd+`(-hAWiO|#bbb0_7L>Zaq(w<}hc)@nLu z9$$YlZ&EfX&Q2_Ccx%1&PRp8C)GdfF}{ORsDP3BtjB=Z)rTf^JR z9n;RTd7A{@My;;En=k)a`+esl^^?q-^(WB{p9p4?v~PzRKWl0H^tin7eWB(mYGrl9 z+Z$iHWv$KIWboFy!!!B6rz!T*aN=AWqLX3wIMTABtu)RLcZ`l%%kyuH}?gWt06x4XgHZMUy*-%~43*?UZyl4dPE z)XXWxC6BM)V)GlwDRrF9ZGc`RST0?@A-{mbNTf*)PZzq4^!jo*?W`Vbj8_Vz}b-p{r zlX(l-X~=1Y)Roq{yh+Z$xZ!QU|6S_*dHrng_E^uA?t2urs73ch*jm$!5~C3U_Tlzwx+_5ICp+wJ#s zE_fU8YD(UeJw#_u^Xb-jT;4RRsrT^rt>{rE?=+PuvJZ_WFxiu<01Oh&Ud z78UN~Sv^)2=B$=Z<}G4ZhPThpxT(F(+kEi0esP`g7HdAqyv3>_aVy1R)n%USc(Zn4 zcsu;nhu^Vzy9c~IxV46Ov-*$n7E3wv;f?36(Nee4T8%$D-mI<+Z^J*?_-&iF1>kLb z|L4-Zr<$FH{F)-`wdr^Z>B>5thEJ1kPHFN|o^-rP*|-aO3fk~?#S_n5W%ITWy!Bt2 zk~h{3(d;=SkMCMNZ&HWSx%+k^`K7v2T*{M39pN zn8gG%Ig@$QWKenxZxaT5>$wtdEB4v@oa4?t@9Z(1mb9OIR)LXa57sem$}yDvD5S$J zq++$!=89YA-@gxkT)$V5xARN9-Th7l-sUf>s)#J&WhjnEnRQBc)Os) zTmNCJ>-Ra8zHw%pnm0`G6!=w1a}_;|Q`1AO@koC1Lytx$^TrHPhpaAn3VF=agp+oE4K`>Dg*ze>ElJhB3BDe9K} z{#=>{V@d~`9sGFL%JZojgTh^Zx zX-ZPvTzUK10Y|^#@ODXwx5-UkNPbWGc2i5cjun@t7b|~uyv6E}c|s?mQa;qA9ioOHOu+vO$RR<~H2{GPJ& z-AU_t%X-%dCm7{BVVAdzcJ%JA7nZJgpTpY~CEk{dE5lpX6Ub=4UZ+z$1>c=Ye&2Yk z2}pIBKRe!7y`T+mJ3sSf{PV0`-@jd1;%&sUWq4z^zM4JwuJ!eY)JneJLNaT15i7qY zPjJo2nPfW=h>vDID*RoE&-)gyY_NJM9I^I}CsSR&K zT7Ujlhqr4=yscUlfj4D8Uf*A3owN3N)3BtOlFM7H%vN>7+ly1b?f(6cYfHQ>Z1-Zi z_ms7y`<8L;k#XkHXs*?0DQ7rutS(j2c%-$A`{tqF<&BvmYQx)4T90Yw+;7*Fc$+#k z0&fv}&g~8i=dE6Lohxr&yyb=uI=ub6#9O~LDS6|4mG$IVi#nukb$R11sJSC4PLD?G zc?+y)?zhoD`q3JPxBrxQYt-?j@b@&Z&)n`bo=!4v^|&MMj#|$fSEbN~w|oD!x{brz z^(Efc%!t5Sir%Wi82e1-jn1rf)@+CKrtHx0_R~Ku@9gk)Ly5P0*Zmjq#6q&V|P9>zHV#+gUs zk=8Qq^2RE3GEPkowdFCqeX85K=?-tVlz3Zz`^(jRPuYK-`oH;p%V^Gqx8r}-_FC|^ z^4))5-v~d?;o*5@fMY|_$BdqO)Nz4}HKka2cp~lv$FGKHt6KxyQ?1VbPzksqZ;ip* z#7z}=3+X4%wbtIte$&Q&;_}AKwWhP<&DyEq?Z<6KoMQ9V1iba?wZVN*tvnUat{(k+ zc#A#(!`s~M-`HaF))c%oSr~yge&fh{hH;5WdZ;yi$)kzur_CKPv!+}6V`<;I$@iPI zyA<6T-sZh^_}^^ant`_suSVdFy+!Ol%DpP}qvwsgt4E#88+Rnlik<{*czg7st1q*8 zYYyHX?z3_Ccu#$2wfpfVO+e3E0!y0DzTs`)jo)#;e`^8WrrcYGH{RIPH+R3Lp)TX> zQ;O3gX|820YOZD6cWv<$n1PyAn3-{3&HU6-9@Y)f$-GIK*_Fmq@EhJNZ#`~}DZu534cx!)u1l|&QNX>(E`my5q@W#ndOS{l`q_t8! znYV})4R4cYA9AS8TWj#P`K^?^NxQ3;xSqE>cf`}D4(TD8w~)-%TEpAe-<S8d+fg14y;Wyl++LCuL( zOBYX}*1F1K8sWStr(}40^uRCU2M!kh{Inf->%Bcg-uUZiPEv}~L#^>kGhv*0-207A zR;OkLYRSX6q*<3*@=JB;rze~@W=_zCw}tQd<#qP`b{lwWIsBFU-_wMiz24=JJT7m( z*)(kU=G6Fo>*mVa4@ducw9Q+4@b>bf9=vg<)SQwO_w|JMxmMD}Q((8$(kkD07CC); zNXMI|b9xHe+;1(L_rnh;FTST8z}wCxmbo&r}eL0l;qBQ+elgF!-#UV~iK9Gtv+_5*eVX24dDsIr^Glk3mp6Wu z85rle(?cEN=UQJa_2BYGCp9ZFPE8MW!DD!P&+i8ew0Y|U-tK&&0&ko!^J{d-T1|d> zq*_wEcnZuQX(@x`ueG?#o7ADSQ^u*SJchSrGta)l=B+b$Yu9Xx{ynwwH{5Ua7}xVw zkGo^fdfvDyg*Ln$c;nB`vU%$Q-nQPAlD7oTYl=gu&+76fb(P|jD^6R_n{QY27qsE+ z=ieSb%;v2tcw6>V1>Tr5i)rMUlY7zg##MQC8>^O{x4?>qw~0SF`cpP<-N4)Qman<* zDgSmBzgMJYZ&I9jsJT|s%t=i@%-Z(eoyCCB?0ZhPU^<{hp5Y{nit_t)7&UH@@kTw)%;}tyGI~OsvqP03Qlvnx!=@HTqImp9qG^#X4zo-4zfG-2*EL^D6Nl#?E6$f;PPU-!;8Q+Pw7!Z%c1`eb0O9vS&@Q;-|Zo{Vs1A zb!&LL^Sq;eV)J$fc$+vaGH=pnB%?j1CLx)zkX?kV&0At7F zb?edZ@>Y+#(Da-gZ&r1~+cV$Y_L9xpK=5|goC>^2pBgD{_2Bj9YI0tSpP3U<%jHc| zJ)~l+wVFQZ$(6S=Hw-_-=4}voYrmlaZ?UFSYk6GWG#&CBHU3)gB=Z(=N9KO(c*&q+ zZQcfhx7WJAwR^m$zH>{*Th{#e?n2W~GH;p;N{``f;LF>dw0Ro>-kzLafwwFspzPo4 zE!O1ZuczYj#>|>dG@e*FbLFk?V+U-rc^eAe=54OPn`SaudpLROkh0i?`^xsd+K{4;k?z$%^%L2a;Jv3XAfHQew(-9;H}3Z z58jk@%f5q}wXr-J_QHA7RLR<-nOHb)%3T=Vem7{$zii${fVa1|c<{!$+$puMhki** z`6a(5o)2%nooC_K^X6OKnzP~UJ)?WKw0Ro|-j?@$yT0C2>H7!Ycskx#g{Q-~bqZx~ zd-|!lmiZZ{pPC+OJ#WlL&1~$7y5KRqefPtS*V(*{0&h!}0V_Il1GYb){;F*GY>tCOFa}%LHeo`*JPGF%t=kZq&0qesFQi4 zlhqlgrpMBTw^4sS=xCd_yTDuP2P*Jp<&1K-R4XT$+;hK0pMc@*heNlXZSyt;ylviA zfj6EJb=J&;)Y7btKJAL1L`_#&Of~s_i&e3(JHy*k$M*ZO&D&V;_VAGH$?qxuRhsf% ztvGpUkIc`y)Y4ktc<~gNnc6p-uV1RgIH$q<8V~(0Z&GHiqKBH*=`Y0JeNWqe`ZSxj zap3K-hb!>LPbKS8v#zgZerc^#ONvW*_;;Dy`;DDYOH~-Bma1#wvGTB%o;RtwGH!V5 zbkV76ZQjO%w|P4%@W#$rmzo})6yuU+9%{)SEAH~nQ0khQ*j>UopSPT8&D?ZUHG zpK0?p0lbYJu_O6C_1#^)_``Xtubq;*j5;^Gec{5vuh_gz1aCc-Rhc*GGZiuuP0l^M z#hpN5=7zVOJzlug=4}#q+tGMu^8KbcJBf{uwNlPl@pRL!){a=0TABtuyxG$2TB^c$ zp`zh!)_uSKxXs&S@V4U43cP9FZpp*AMzeovX)QgH=307+lb60qCpG=l^zf|dk+kGz zPK}oGySzzGDbDKjSaHMK)Ebum|Z4>uB_1%%wL##Yf{*ZWF-a_f4eeR`;rx4Ofy{t{gTZ-Ko-nu_E;|%+L zn*-iqUl-Rv3Z*d-d4A7>b|G!R{9=NigRsnE-dE@zaxzH)de<1a02atN!%Nr|tV~K6sn7wuX31u}81(**L3Y-q;B>Gx+LQ zd01DI)8$RGidCo+)*9aWKla(xHgETUx2~O=CBLVvm9g|vzf)2d zYZq1?!`mMg&bq_qZ2@@OHZvt}ArlK(%RPqVad~4lYF4Zl&HhQ6yX%NMGQ2hJ{blFp zsV)R>PrsCsw}^d4zq>TOWjgt)>`2;muDtbm;2?*$Mc{3Ix90lyl%E=DcUi~zt1xQc zy2-qSWM(JKLrssQL#EAIae3pv2V|99{~oZ%=703E-_yn5ZQ@)H-lWOYV?3F+kR63^ z5>i)M8<#h!N+E9E(-y-I`D}@|75nUc&hh7-clMZ0OWIF9t8nZ0-dJb2#X8LFa7#Fl z+@4uZDK2izZ0*$Gw&N4O+Pj3?UPa!{FY(r>XN%1CbYnn^>?yEkv#bzI(< z**ZJJ+lN*^Hp$`bf)a15@2S9BLjH!c(|D}j%C6Pqp(jM|e!NK&PUcNom14Grx4%Dl z5`KK|uKVr65^oD$@!%~*4;8s$PR-sm9_EyEcDzX)N^z;fSn*`uSS7GS!`lzWUw)W# zzg<+~ZP*!4ff>pk^79m$^^Z&Jl-#dGDY?U!!Ae=oC(w<}7#ZGK=6Zx`jmp6|UBi_ctn zJ97HOgPr&E$`Wsnza5#kJf|8`E15C=#RqDM0pCvOr`FVxdXVDDyh#<=m6VzBg5P}q zHlWq~jSg>Dm3W&mv~~D<>f0Opq?YnXajx~%$-Mby%c8?rIg@$w-D!f~@OJ;uO{Y7& zU0vd>&m$Fhi?!>7J{!K%rYGyVad~6sAzHd^7^l|M%8oa7QqYFCTNhn(i^JPBCEi-? ztiYRQN4_2nOB#=FUHWU2H+C3HXUCg$MuxZjXMOi@hqr4>yuCc4P567NnG8P*8o#fH zf9;f7%H#6J4ymPV^stH)_szqcA=>3l>QJhp@fSRXw*@b?zue*Nx)N_|mZjuP*+auy zsqynfs5uA5H95JK{^BVxLZ_sC{ZbyrH94)Dq+zPeBWag6$r%za?85MN$Ud`YJG}k7 z#M@I%+U9e=h3v(5+8V!mzvbCsNG;#B;kU0 zdFBbf-`gdhPQ|Q(R{SS+s!53X5V_-p7%66 z_N)^rOFlgbr&)0=^H9g-jhQWNc^3db*#;^Kz!cPSA6tv;(<RIx&dzNranzez!iCG4BncwYrp3~b=lLLNWJixj<*!M zGrXO-Vdo1rZ%x44x+#%)OVMY;BdR%NjfXu@b8R}tm?4&yI*gS+9d9YB8{Rsda>5py zx2E82$?7t^Wj%oudud>jy;1w-%!jv-4t;kd`CZFzVzjf+PpObZ=*VN zsP20z?a|sx18=e7dfuwlVceaPI%!8zT-k}??d5M?h@Zz*{Qj*ucxykygSWtbyqL+l zmYz4ww6pG(J^0olY010x&CIR}+VFPN+dsX~zTaAaw^!Dd;mxYIO4C zH5nw2rgM#l`9u7A-dHh(HoR^7&EHnqytM>x4|VQX-S;%bzP#>9DSz=4e9uwJ7AudG zKO~+jZ~t@Vq(9lbwE}OGXGP#mnl^t!kQ0;Q8V~=%B{kPdn*QP`Fe0sDoSkzmH9cG_ zX|AP~{8F5Csa@XqRVkZqywJJ1-`+gvq{%jKw}Q8+>m%^y+Z*@mtEF1ZBY7Avo&qD( zQZ~jlTFS|_l813$&HVZBR_M_1ws!g@_yHotUoW%834JWag~3Zx8PM#tiIM6Q_r|kjL=$&fW*?WAk)!m#oX`bP{NttUc9?qL`M~1iQ z*Y(@a=B)#G8_=gqy7yGsLzLf}&7xL)@us|C%1%31-i|){-}nK(#orI<2;N%V8=1F| z$z;Kml(V|L#hQ{PPdIN<6)SG;w?97e8TeRW!Uk@wqF0XY+PDc-!${jq)b#F;+bMev8#f)_L^2 z@zdwqRbj2+ZPNPpylV5-9lWi6$Aha+na0c*Q;(myoGcaYi(TK61pFCBac&7~4R7BZKJUl&{ni`2y*jc2Z`Q9tQlGDshkJ>oS&N$av*V2w`R@@J zXD8H>hjD7Im2|rMO{!9Xw?Ccq>9#g+cYwFmk5}ML+C@m5d(r5ST3pLo#ZzE}PK}n< zN^vQt#xHrKIBTWjjTu;lnx29-_uGxrhy32=tq*v6vT2Xx_f+Cpid#K2+=sOm&+dN9 zb26+&&1r}8#%u}N@HY7VtNv*7))%}@zbhqgtV`ypC65%B)=GZHC7pZ*rdH zW`1hlwe)Aln>Aa*+m9RdKg#BL6atv`6Xt!2;T_f)eZjVDF_UW{papeG$~wYCfHNYd$eld>hm4R6!${rC+w zZv(*FYvXd_EuqioTuD?go{qOXcM+>ruDl&N@Ji?BTMh(o%b%^l8^4RtXw8&(?izm< zp8R>^t~p8WP7{|r%tP(+rpX|A65{55>$3Hl*7p522)y0bx>wxyl>Jb10Q;wfa2S*qw8kJVc;Z&Fv}P}51lV|d%x`Pi#%-o}ErH|Le%P3oaqaqc}< z55-f+B6F+`rQW1C>(Y~sH)d06!`oBqo^byD+c@yHeA9n5Z_*@rUXrI;ao^r@<*nz0 z9=F){+j#IcvsXX&J@wsD)|1!tqw%oM;wdm9Y2VCJo@&Ljc*Sev)W;BE1}5qPuemgP$NH8tbZzB##;nt2#^dE-|@sAe9i zI^({Y`Kg(Q@nqiUWQG*l@HX;{6YsZqn*!cuzMd0rDf%zVZLB;lZ@!HA`eSwOnd!jUy4g}lj1IKl9TaR)tQHywWu{7`l-|L z#tf`ZO;14^-kSVw!tFM1)4*Go`#pHG@;BT+{47vQ{%Xb9TRPrCI%oBeJknYze>iVa zMJsN2d-jpHoqxYD9lX8%Rs`Oxx(yuGay%|?(plHS1ajr=9+=&G znsE0vFi8)!?@4j3q?t3EH&$V0UmYut^t0yT^2Tndcm4nMUH^YaGp25-wA^5D&P zlC|s)=PlOmvc4PR@|Lh`^Pa9ctK%s9ewzv2W^B)iH%^tG3--^rMoViMmvr$I*cG+3 zN{UO}N`5KMJeq#!XHLeGd6TLz12fZ4UGNy*F8|*voWGAW3%m^msoW6B6{_tsQs`z>|erZZlJl*}qPN=0S1&`saU!Sl3%;s$#c)R-v z58k9&^%yUnf;34f9&4vr=Vzbkcw+{oHoX02#Xb+(yv+x11DXv<_numJX~XwdjBEPT z_<1_?Q%kiZkN$pR25PAyyJg(o`ypZG%X z_F~JS>E2Vmani)Hh=F)4$L5a7u&t=G)<`-D>NSe2V#-o{G@lziwo75rW)KXo>>8GZLTJpH} zo3u*Gz<7c-ynXY@Zg)AnU0CAn+1rMvdrvhzC-m0v+ruHAkRH;#ry7z&cI2C%6QE`f z$-GIcn1LQ@dJ5X`*0WW=!yVo(D)DyTw3NJAwW9EoXU=rIWznH8W4=AOy!mEO`VDUv z-nR(<+|_kF_Q?UE92+wabaH|gD%;$(xG|0sl-9%|;NE>4>fO*VR{nTMJl zb`_$TpLrN(UH5+DSEUThMvt_%5I4N7X@1&%4sVy1cx(PrO5WH{ExbPLhnlmZc0b)* zMa@+qx>{3mdE@S=HM^7ig}CAE107yD)Zy*25^r0&j;x*cG|v-pdDBe09z8B^8STRG zHvRqY$B(z%_3MSpOS~t!= z%>DMv`B#44;qBig-a7Xio&28ivp_BN!@X$yQXa-Nc_fb%kIS2;Td5PS4bf75mp93& zsm}AY{D!xiXDt1R!`pvKyfwYA0&mPI)nc5QYpFFJ`l&f9YG04bo0Lt;%y^!f(__!n z^til*bjvE#g`9@Bue|Yx*BsuiFY&hVjS9T6^N>D6)@pjVw)-_Heit;Pt9n@*mp5tG zg)=g|jlJgHE)H)ulz4k+;9be@sg%E7;u5Rw{Z=nK^6gX4n=~tBMZ?>OmFG@!c)PL0 z+vEr8jJLpVofPl9?pt6snhYs z46IViI#=FWZvMqKhqs$cyzLk=M*p7H2mJ+?=Z>R2A;r1t&81NKABoGx$tDz%h>o?6rF+2t*xZVhkue(BP)ZQh!L zw{1_B;Z4&|$Y(*5U*iesLC+gI38~0DdFotw>)zr1f7`sZ0B*v~#ept773XHIdMu)7G@-rv3 z#xHr2dE-1a6VLzQ-(L@td_R+;`M72P0zJ+zvZ)2Qm2_u zimnWA-#c^hKW*M_1#jJMolu+aX^Q=NF_{lrn@%0eg425ZEfCKgSTxH zGUSaDpyrf(b*wz;CeP~BzPs@Cvlg|Khn{?RD^xVReeT#>@DH?#f4NA-)X;q52;qB`0uKHh_x7)y5r`2V6BhS=Q5A;y``hC|*`89EuH%)cv zo{`ov?yI>AX)QgH&W<;$E5qCOPrGN0&0Bl$*0#f>bnmIOJLyCuf3@PpQ>d42L#C7+ zZ&p`^x0yHIh`#_={PjWy@b=2|GQ3&bmi>JOdaSx-{q#s{8TZx9Pc7vso`UpMW}v2D z6DQBoS}Bjl&$ZNCn-6b=iiWokEWK(n`RX~>`fQZnM^v~d^;?3VtD)GE1OTSdFuk+9#~%oy!q~34dx330>Q z5eE;PZu8b1yuIFYs{TDqsMWxjFX)U1@ge+jBB+zS$BwG`tPicTF3c zw;tea`GTByOX#yAt68i~zTYCALREIhi6!$!XP`E`ef7C7|IFsCCwQCl>TbfDFCodi z1x~(!G2gCY)n%Ssc{^kDpPj!S(hIx|?K3T(_mmT;1)UFXAv@Kus`2E*TcHzkzg@h3 z!fp2b)*HOFxpxn57v;mAhK&$Dxt6t(dDHwFLE|Y@H@r>!$p_zS^L7V#+p_h)nm3*c z&x=~)p}*F2GH(&PHN2hj$t}*`59tHmp6EY)&wE-t5nrx+{hE`Na!PSc&RlsrZAOn@ z+xJ^v@HTyE8Qyq*N;{=Tvunx2Jk*k36W8P^o`S~7iqbUb3DHtNA#s;CDVwHquB9%l zHTT>3mmKW;{E&X&ZS-4Zc*}aKntf&MDV{>sRcax>%Ui6jEKUq>`(63QrS|>SAG~!L zJR|u%wQ@H6N?+4+$XY3XteoMz#p*=bDdVg~O^>9LdDCRmcnZ}GZx z0K9E^umW$=E;MoLzOBeA{nSz|jYnGR^2Th`QU)o`wLB@trL~$kb4q^Ja(UBakTNI4 z4R1et;h1wvz!iBL2;Nq{*HSY-{nRdRT&2-cMa|k+9`-=Z z{4Q_&YD8^#`|AI^=NC3_gTUL8VRt9LrxE+{;xOyF;k;$tA@lQ-!nj7~%G;Ym-*)~! z`C#xi?$H|MEo8#_Q_wi;-IFVCpZUe*?d|(*2zcw!Xy%^#ExWspZya@=3}?q1cT~{k zJzcra7w)oo8w%c|%n&eZVs_J--F+V|Ti@Yc2E?Am!xefJyUkINgGq1JTh>(}I@ zC!9BBhlaP0e`>$GZQkw#Z*Pu^%o}^7<^*a%$K@@H9mSenIB&|`8QwnkgI)*Pyp0BL z&paEMH)U_pe~-oYbZVcsSd&cV&3Y3X&P9qRbY*z!`t-$Luz9-+ye()wNB^Gso@qjF z4f~=;$}h#a)>o(FP3lC7vn!2tP!*@#Re6uMhV0gRoxBIrTc^eDfZhbBSZ_0kWxXL0Y=e&ov zyusuwNIEqU7YvO7(myxB6{wc)MVA=}s5_uF{z z_UP0Iyz!^)@jC}+w02k#_tL=+NxXj9%CMj zKbD7esa@XqRcdykiAx?0V_Zwk9waUK)A7b^)U0mB4R7A2DvHT%B&=T`qA^2b?2IHLway|V>X^O0krM2`>ySy=jM#tJ6y9&`#&n|CtO1q$kT}knR$MAN>=*C~Pd7BR27IeQS z?t9Aq*&8)I+)EZ(s^#()t2%dD3prihVs*tlg7uGMJfOy-SFYN?9kDZ~wLlYX(6^YifU25+M` zm*GvS#i>dj-?-GjZysO2d%v+0%}ync6lZ>F-_Cvg$-MbyOYj@s#+>@%EcX7U4c?Y-DZ`uZ^w=l0CXcUQ zlT+gf=PjgjuBFy=?(5O>=9}5_8{Q^Q8*z-y+Z^yVz3;+&-czeLuX`_Kt#q2OIB(Jnt%`=Xb{+aUKQDSNch;*F7;Due)gHn8=dv8UEVUPE7_tze`dAkR^ zJ$!!|-lTpspG117rJRh@PfbsVX3p$*lRjmv9DIjJRoNZjR3s>nFCvby2zFZXP^pak5CefB=*N9Ue*_Lxpf+D|^Ka3`%8esA(~ zsx0Moy@li?&XU&T*W;$iR?8k2w~%fVW^Qm>JZ~fZIBTyWZ|9eIoAX$m@fI>GuHC(P zBNZ-h{3>S>s10xLysz0orR!}MZx@t!8`5}*d%tP=(Rh6OXHIJ0JQ{y?yoGcgvR2x4 zuDt!>@Jm)Xyj@u0t>v9%c#}F0iDxm%kbctfrrCwY6VkcwT9-FXh7`Ln_uI|;pR>*3 z?V=KI>!0x8P5KrxMQ>5Q-wUam%$u}R%}Ir<)#TTB*h4zrq$DZmwBc?4Czk%x;qB5AZ#!Bpb>Gv#y3s$~342UkD<9sn=-fA_%bRZorQh&2 zYSCqV9NsP~@wR$=8Qz%Rmp%HW?=7V`>(Zmq(ptu;Ylk=Pj?-heAv!K^?3TLFq2cWh zpT7Mthqud1ygjm_0&gKZ4Ibrnq~u{_dV;_{Zzt>Nw0T{eEm;q8hN zZovIoZ&uE- z-!EDBYJ-PyNi#n^j7wVMm*!JDylHkJO+bn>r^}l(6Db?x%*=Qpr{V3W{Xf^p;q9st zZ?`@lfj8EzH7(7yc6iHrVlHnX6{UF=avI)xec|q}IlNt6;%(Dy4O|e;>A;l)k&;8v3AEi$-GIOOL1nVF8B>^XP^7AZ#leOQ{wHxX;t8jeNy}Gk!yYR zp8GBCC$2Dm!`mMQuesRa?b;G=6JLnHTb`WwawYkbdDHAjdQczIinMbi7GjC7hAr?V8K? z>+kUP?-FlKW_s|Z*t6KW zP1;e4-lFs4+e2L5q{&Ee-)`ym&6$oj-ztR;-+fO{CW8CZ@d0J`3)uB#?0~HjlJ=Gr>2Km@rS`2Wt#x?|$;>L$AvrZ`HJ*HU zE9~0vc47A|-5uU;F7dW$ei`2Q-w#qtcPu9?Y38BU_(SS4Kl2n%ft~QH)S5G89%@$C zXs)H^TB$e2=}+cOTEz_Pgqmv$+VJ*v_ZB~Kc)O*<+v}Ul@WxsitvL}*o~%9W)4kua zt}f+gSCYp!&Ya1-`DRY=8{R(j|K6~Fp2LsMZG@lau)Oyp=^oUC!ZrJgQubW?z!4gD z?yGYJ?wx)1`(f#VTlwzKb7&0SW-Zzcd9yD1viuiMLF3eTd>Nph`5Biq{nU~_9dFD` zEmbMR%>}n&;FR$;Z%x44xYs;*Q`T)bU&$Zqlgs?noQYH`|ND)dP_tWU7xZW}^M~`M zsaWXT@OJ#y`&vKW@<-=31#i9kJ(}Hn%3V`)s#09D#}H3;ym3b%9dfOdlar*MIvsCP zMP_F8!cW69yghoz32W{9tr>XRxx|Av=9e`4p{8Go(-Wel{EQb*AyyS;re=QMxaP!} zQ_?PPAzevT3ONmL{XX~P2Aj9$;BChnWq8xpkxBmeGz3Z>JBw06%ZA`0LY_;O&tIQu5}@kK~u)oC!Z`)bvnmJjGMcIGNd3 zGe5PIN1BEdXCBEDmp4|SE_7meyX?nfme}`OEATe??J~S^7t~gMultR8YC+fj{T6H5 zad}HPJHy-Y9Y5v#`t(-tcE`|V?t99dv3AXzwV?O#7Cr%s4a3`%4ueP8_gic5*7%VM zypatn|89K0WwFyNa`JQ}t-s%-4yCwN(eQS|>I*Kid20jSp5NiYo0Wez=56=eDXH__ zod<9K`Q*NhZQj~~w|hrCp8TF_deeL-%sli{OCH80%{-d<6z9)XQf5s&Bv00BNm@}$ zJCZu#JS9&dr@7x2+`Oo{&09P0HhozH-q?RaKMnVx#>1S{A#0gO&s(gDte&UV^QP%4 zVH$?F{$E?N%;xPj@HV{36Z-p2s>QgZxfgmE*J$6hl0W&AuIWVbNEI2EG;`9!xJGkr zGH-NJYpT#wSZjEDal~GKuz70_-ujRB;Enm~U8m!XFH5~lAepy_yD+?c?a;#xw0Y|Q z-dZl-O?dMqAsufKPp9F#3+Ci5sOj<5x$^e=7na}$_!j?sp(A)(-~37ad+NK>TJ~qh zTga=F#l-UAZP$DH%r91b(!Sq1fw#xTR^Tnx^s>&w+kmws?efM9S)98jXBItV#~Wu` z=-j-gO*_wQZ}Zj}yv=&10&kirX*}$W+IKDenmp18y1X$nHLEZi^4?T zm$xjc6uLFMJ$~Vd58L-!SMWA;Vj13I?X;Hj)Fy9POw8piizLZ9FMc=2zTa*K zZ#&vP<-Vt^C7rJn=h|A<$-IShu9=L+BlW}NLpIU(@Do0GqAc;ksfI+J(AYsm)7ce(^N_D7~Xb#d$jYO_5g49 zt**eEw2OL=r{gWpUHkGW`SrYo?59JUGN_;6u*D#3EswZcslt#4e5cO1?rHT z(po9M6wi(~siG8*wIeA{IB)DcK^xxMH{ROHeouRWw?Q*f^2WN-PBn4nq2^k~sp*k4 z{nT-JV`ixn#-+PRigT?-v!8UlF#~l%b;DcVpPxR==B+n)>#{B-ZwYnF{>fmi`HQEZ zoMgovNqHEj<}{=@Jt11pTOjR*xAU5H`;yJu9pG(e=V#)+r-40JJlT-Rb8X!FjhRDq z7Sq%77SefPt>Nu}@1OFZ&08Pvwq{mj-q>@j>BY*!-dx^7s)Vd%S6S$A-jusDy#46f zz2@4y^#yMW*H_?;w>|4JE=`Gk$)m|5t@X{%TE$afM5-dirL~MxGmjLf$5%@^!+BG7 zVtCts|Ko48dFuz>Cf)vQcJHZ%2aQL$U$3{=z27udG~LqUt2I5)Bju;Z<&CT84Ah3V zL2F*yY4g?}ymguvnK$2_GunGa_R{et?XFt8kosZV<&9sBs10x5nttdso3{br?Tw9* zd5bmqSb3y*#^p__qKW6TQ&Okt%GXoah2d@du$GV6ybT0zPxV~kzNef3wQmo#>`&%x zciJ^cE9#7PXLwux(`F~xybS_xGZ&QMjhsl|g)+_`TCLH{Pc7x7$5%5abuw>sYAQ;8 z#;KW;6O%mj`)Zdr-wcx9>ele~;-A0K(&lY2c)R=6GQ4HMe-=5VIY@E#kj$Huq1NKu zkqOG=mf}neZ$~sb zy(xLidP>rMrMlMLUhzbdd6TM0UB!yCE_Xz&sYQRTyv_U4QHR>R4Fhi* zwx;Awvuk=dNye+ClX_2NA&YN$C{GN(h%0myeidE@S4Om z>LgYk_NLLS>+;47bTUq@^cdcLFnZ{d_Wd>nybXHPgSRX`7u;8#x_AoQk)+ubJyM(= zo)qI;tI?XZk|&urX_XY`j;ziNZzpYh)A|2p#)7wQjaH|--xBIJd>_SreEnRj(Z2bU z?>9}wdhsOl#$9lR)bu21!`m%$j@)S9Z{xt*n5mQc4WZ?)#%!&~$T7~WbQaLAiB zZ{xw+%Exze-uOACjy~<0{gmnNw~#xbFk8dhRck)j%;s$Zc$?pJjr*SRTr@rS`ZYP} ziItyu;_}AK?7S9qI^HxpvOI>jE`2{c*5+*@cpG$A8QwT0D`(AqirKS9$K_2^v09#F z-Xfl_;qCsV`wh2wn*`okKUIb|sRt>}U&}}yY1+PiDG%f9hjHq3yh+_kaaNRc$XY2U zQlR6!5^)d~2r=*TwjkqIozbzfw_#m6NDd6pa zaTR#e^hOUg_aepV(dbxt;_?=&D^4I*PLdXP*X*{S4R5bpK6<^)+f?v&*Ry4K<9p<* ztscsLzbECC)^hUFTE?Y3jJuO(25PQioSGhLjfZ|oGY>T$Um3r<@G6G^MQLE^odSYhf3fo^$1`U8iNe?fY#ycxye$gE#)mHEQmfpFSy04>kQE znmMVNvv>;J1vQ=2QZ}xYw6vD_xt4KiO&+oC! z-eP@w^!S(%jZ1}*<%)K^}JE^p){qBgu;a@mrO zHgB`Q+w?UNc;oj2)ZDeSFZ!t^KjZXM(<5n@F=n7<2F4{#e=X=-dAoAnNzTuUo({2;*X6{UvS+VJ`kp5J`STVsfsnP~_Z#Pypbc-QF1q9x zo40x3?a3}LCBLU3J5A^WU9Vv^Wn|6t5|-Xg3BAfs;Q{-8{YmnfAqCBZ})(=J};Nyjlc4e`k_bj`_J@K zOZl^oyS%Y0YE37;eoan#n4g*+YI##Cbzh(Sp0c;y zNxQt&!tRteiPv`{$-G6ZXn0%s!NK$F`)wh3Ydk+CZ|svg>r<0zNpY#4+T=~@HYA?R zTS#VWt>NwIOOI}A^R@`Qt=^Q9H`daewZ`L{lYU>F%$skwzSZ;e=gQm9+qQQ8JZmv{ zTh#02y!?@(9$K@>~8#7zl+;30)^*#9Y>56^! zKIepU&pUferzPztpH=uSVC=#&xJm6;>Ak**$a*b%rk3hvhnuFl#v^s^8+UQ@&7kxf z+@`J>`cmmR-K)sk`6b>uZz;o@GJgYaoSvk8bNc#OSF<)d-ZWQ^#-rJh#*@sOrW1Oo zIp;#$@b+AzS59WLNar0?d+7)vBQYk@OIs}6(4ta zyST(#pZm-37O@Afw=&PuDb7EOidp0gIT4bU>8H%_HsPUD?{s*(q{Lg(Z63T?_oo+2 zoEY;ko~MpGdG1K!jUMW(cb$&66dfAgzJB_{_~)m)zJI&4#M|bClm!_0OJUia3-5K6y z%=rfXxPI6Dc6o`nY1=FC#+=kyOeX7`>_0AVS$FQ6-`y#x^UbF88{Q5cc?^CW*Dl_! zDDig3@Xg8ZDLYrzP5pgcNG;#Bad~5PYTxgIxHi_bNm{yHYq~0U3~%#VzwkZhJ-xET z+xAB*@D^*=wU#H{C4sTbNc$+gSC*CxZ$V#=Z&j;s}v?n7DG zZ_@NQ$q=pQE$dD+y9ntaJKn6W%zN5?MZcjAZ#S2CTk>3u@}@b35Rc0ne+PhCs$xyM z?C;EIJW_tf*$*{6lFp7dtD@oUd(%%_>F{<-iMNTjZQVWIQ)};KPoEx_H-5EVw1#Jw zH?E=%+_mBDg!3<50Nz%<`|s-;;paK@oE8B%e&i(WyBX%;S}9JCM!Uca;?joZzoy!*p z@u1CHQ}DL(?lQb7^H=;Psh0C($6KxKI#=G_{OEHB+PpObZwp`C!&@Ev|1u#A=gQlY zE4NLyd20^dMt6O~eNTON=lg2VZ+#Dt`p#rdP5;GH;3{fn&~&KrFehvIu4NucGk;v( znAxgs-qZb#Ty>GnTMO{kb4~=_thx=%NWDpMDUTFq9%{+Yc<~e%k+g4SDUT*D)slW< zGyT-8?6#iyA0eYKu9X0Ekeus2D& zyh%S->;FGJ;oMun+w%Dyyh-=46lZT5kH#-ix zKKEN;&W5-9|GM8*HgD~~Tg%rxc$3afir3oY*=Jnde0S>WXXn(KJh}4L>x$;q|3`Vk zxwnD0^?kS1*L$j&Fi(V2bWZVH=cX7PN%xOtpj+Q_(m0Yi^W*N-W$G~l4@0#H>oS%I43~OJaKtrW~-v%?b#pw z^=6y5j^M4wz_;t`J>`BwbUwUUCsOfK5V9j_ZFam_6%B76z307)Y~DJ7x1IOz;jIdP zKO|2MlX;8SiQ#SB?|wGT=B+b$TfHqZZxQ>?^ft)nezR_XvOCIpV#&Nk?9lM`=FN|f zwt4FU-WCseC%gAF>q%;6C3$LxH>tx|ahJD@&c*Qdu_Im`Z}Zj_yp4W1C2tw^T=oX= zoxG;E+Tl&=R*L&h!{yC4gVJw!J8JXh5jJn#z+1;16?luaYh}+-c+=D>ojB|B&N94lcgk92d0?Ipokb7%^QM^qcS@~UD|wQ6lUAjO8{WR!?#Ir1 z+6%mGf1(0!zTYjdPt96-sF{bF9!b;BQ=>nbH#(`AnVKF+OMb?wr99jb{gMvnP3pvo z8{R&0>k+N(`>i*4d!yM-_dR9j%&Cdflc#2%niFw(V+N^n$s@&?NAgH}!W zSA8>cw#-w|hPRf-9{hmK+a2I-{g?{8u^-lA+*dOXwQnBDugU51CS}varHWFVIW->o zeYMM*Zw6(DhPUI7>W&|XU;O*zeZbq2r_1nWahv*kGv<-@7|X-_%t=j8IB(2M%_>ry z9!Ya8HT{fB+U1Q-W|Oq!x8jDk4Mz_BzJ0&-1#kDXYV^_W*EA3nehO00wHB}6ohok| z>r6>qMrUDoJFeAX-E7|afwxf;_V8Bb?}up4RO89wWPR(Bv`jx`hPNI4#&xxM>kr;K zuKcg&&G+mieyWv6Q#YA6O$LppFqwDnx1S!e#pZ1Qc-zvZaq@f0H(H~4BJ^wG^hlaH zC9PTOe$qMBkj&Cr#x)(%!?o1(P}Ae`#;;1NBoF=61;62~?eQ~zZ1Xk{yuCWP3~!t~ zbM8*sT>B>Od#cRe@GBszZe7G(-fC?ZzNaAh z)A5#K8iu#!qxU+;zTbv`w{FwR@Rp)hT`csr#Pt?CM1;XVP#F7pMLI**ovB zJBw;>BjT}qN3n|vVsEJEQBgz@K}E42^@w5t>Am-skdTBVq?1B=NJt~SkdRI~AwYl> zdVoL(gwRVU;oHOJdiQ@|DMGxI!q@0fqMe%E?`v(}n5_nvtZk~g8Nnq?a7Q-8lj zPQ&o_on4Roy#1bz25(E|*5EC)hfqH!Q>4~^x{K^8k6L=(BC8wTKJnU9r`o)Y0dHd; zufbbn4=wiz^JtjVc$ib8nX_?tV^`dT6sJd{<9SndVtD)7_lCBzc^eDfI`?jy-+QX; zGc%K#X-l5Q;Z5o=Rovw*=3Jz%N;wU0zy8?oKVb8A9e8_sL5{phz2y-Poni63g?1P- zCDw9zi^=R+Yj`{8plKJ|yp02ItDg+ujk{o7{=W}udQ$0B`HSZ*)dWKGq~%STe~Asl z+ZPV*|CG(!cKL#IJMd8VdE(o#QCp7JSh6*a3cPAz5T+EA^@$sVNq z%;WN=$slF+#0_tYKKrW;Hg6Na+vvqLc=P11`h8a;_&II%!?^x_<0@ur7VYvTb*|Z| z$76VVdjFla+PqB!Z=IjXkvC6oEv!l&DIPj2DUY8wX_ZvHk>f6JIqlByw$pdt(bnc| z5_o%ZKs*0?Dorxo%j`<;|L2(4AJVwVKK2 z$D38%@OJRuA8`Ks+jQ{u#7jGPtMd1=tVss3SL9j^=YHNa8I(IRygl^xUGA~(x9h>% zs%tv9@2T`_7RITiez+I`gGToHg7Y)+p=4$@Fvw_oGdZkELzVSca+DjnN#yOFW8Sp`*~w^O*W6m@HYRq zum8#BZ6@O(qp=IzhMTZXO-Z^JkK z;C(i4v%p*XJ8JNznWTn)dZ@WCDX#IbPbm-M*?40Hb|q=apDNB>OFj5`lRA;IN&ZqN zhPS_cf6|X_-e!Zhm$%j6O{y!!H4~85hQ_76Yw~2{O_Moqk2GOUgWAtq!fp+3W8VJx zf7-mw0dEhE>6rFCP1t|Uw>zd*@w`c##Oy+|wou;g-!^HI&D&h?wrtG~-u|P1|4l=z z(qrzo&C8$pqRrbp@HXMHPVW82+gQ?45A@T+xTH1yRJTO=Q_oIVojRr>^HVdw6qh`q ze*OI>Wmd)wZ>!hsez?useDF4Md==if)4a9jG`U}DsW*D4Sxe6wt1tuO8XdD%lb;^$ zTF)CZP%|6j)FqGM?X@)*KV1GSV*ikH@!`|U#u51ek_Z;Qd(f-A1f_MWmg-o1=Vz2zNeKT<6{ zZ&C)vS&VL7Z&C(j z-0=3VFFgNIo44iQ?e-Z>z?*WvRk@PZZXdkylSs{7ySzzGt_szqoQAjcn|t0?;q8H4 zc0THWqmMaqVviMFuRpAGK2tX|C~kR-gd;2;LTHsxKP$#)sjd`fEf+U_HK8`REkEm^ zvn#mmROap23U7V7b#46TG~rBIuE6c%decm%P~I+jW?dJDx8o|jy)i3C-uR1k&1C4| zT55VUT3Wky{?l)93 z=k{^GaYxcL{Jd#0NSUp;;q9(NkK51T?UV{{Z5CGHEnyEyII;GXBxmuw)oZGL-a=>V z*@fZln}0a}Zily1E4)3nr3!DF9z1oE@Sy1-#^dtV>^qXCk(M{9N-1u5yZ_6_-{+Cr2h51MV^b{ZR>v1T06X*Ug2%Zk_5a(_Stfpsq#pZ zXWZp2w0fwY)u~hEVNPm2Z(Nl@8{T#ufBP_pw=*idUG;1N-ZJ#o!dSi5y1Yr9w>v_w&!&K9g^5}V!GAJit?za`cejK>H z_4{vURd~B^c@5r_wOZ~g{(cVoXPi0dVO-Kuo=|@_-cnU(9!YaD^l+z)(@&4f8&^r1 z8KlfyTk;#;MxFDKcLKQbpAR{^!rLt`RN?KCzGt`aL(Lwjt-391>5(+^Q`65lwdB$C zQ$7V|lUC6~t*I{MOu09PeJ;sluZ-QBd4aWNDv5m8PnRv;XobXjVxcW|KTI@l^R;-eS7qTB}3D+nuAHb>GwTE4=j^ah3a?T6qfN zfqRj3Xm7m#spEN*I?N-^&Z#xET;4)w$SR(-hPU?)UWmUg^VY9VFR1X=dQ}zPn7`Sz zG#{5Y$(i>A@~-RhmeY<5Zy)dd@HFQ=y|BXDqp#KAEp!_E^l>fYIqkRQ9%e0S)^&N~ zSE;!R#w9KJ8JD_ZoPMd6Chq4gRh3dFhPNNh`QnKVZx>Z~yLnV^_dV6@qS-xq-Wp-2 zF;w_@OSn73+wZ=9*2@lW7gu+|lpXc!6+Qh&qo2xr~*?_ATG%-^x6u6iEH0g*6aAn?FgSUIz^z}cep*zj{L}K!D zt)DkKr3q*}TpQ}=T53%mdeZX7%2l0$xY0gat>AB^c}pFCyN6YqV*wA8P4DT;3#SWZdx9@wqM9uV4dHnZVy}C-ZJLV~rw@rJV`)~VxyAr%D>^@+}d+KM8XGadH>7mwm=$CY8 z9{RKK#ta%=>fHQ#Vcv<~?`8AW1-uQJQ-e3wlBU79MgoT=6_Pg>rXS+l!Tekl**h4S`ahjxG7=B*oe zd#Kky_dR8|)S5|3o>Y0H{GoA|w?^wQbVrikWRH&!oc!`pouZaTo`ttWWv*>{lpp5`&VR5>-LpsBTe z+;94wk~+<*3?9SVgG2Vc!RDc#CS&!P`Hhar<4!)b^O)+ITA?1PA3txYs+95=-rn)W zLEUWLdV{x_{RjKsQ|r&1WM+eJy|hQkqv=6fE7i)zo0N^!sU<(-l0S6U%tP(+rpYXM zn3=mMX~Wyo=U%nW=B*ESo42$EZ_H24T8wM7W-UFEj;WQFH+Cp>mCB>Z&sr{T(oM+B z%%IVwwT8D(toffuY~K2Ux7p9t;7wDjULI*~ntsZsAZ1`&(wZvtFc0H-YnQj2DjMGU ztT<^eo40=8ZP4H$?t4nksbl(-@}!DudSfm3e&bh}O%tal6<5qj&EBXbkIS3n)MWE` z3~w{uv77VzZ&!o2-Ycr`#+)%)+Lsj1qn4D@<&B)MLuzJYoSGg<(=TbRrH64z(;v?p zvvEpRb;H}%&uyGz-*5fFTc?)-c$2<|WL@^gIJL$@Klhlb2bVWi4DB$LU&_NcwKNTS zT;8~f&d>>PZArg*zkOlbT${H6;H}FwL*4h3bz?L+iOCbQR_ejeTOJjqPBgV**1Eh& z6=SNDavI)_{?KcaZQcfgx2JDOz*}ga*5{(-_gJL3G#@Fh$x}WBjWe{0lv9c`zvPkP zTq|ikZ&C(jb;DbSoi}{M=4}vod*YP@yh(jFay;Ap7W&DGnLw&q@w_Q_VR+kR{{!&@ z%FBPAd@y)hb?vbH-cx0tE%zJ!siw`G)Y*7r22F<=4|C?B^}I!PYj}I$#-$tW`)vq# zn|gZ!-Zb~EW-rX6nLKlbt}UN}W)(fu?2vIu(=TbRrH659dL-@g#thU_2FB^PwBc>j zdAohk=4~i=yKY+o-q=qJ1I)v{Fdn1XTm1bN(}}b@#;Id!xx8@>F`2EkhPN}8cEvxq zTYgW6fw$gcuF=1zp%ak&Qrzml>N`tMOrHAkCha10M;gD&8>^>cuhfa*t@VhmpR@0` zYrxy(YXW#ePAH^7F<{VsvR2hPPqg>wCG)+i>vqSnJ{0?l;XR zE2e+V+L)8_^Tz7jDK$Nfto6J}yOZM9t_^S9)_3^4&D#j@wt8FuZ&v=A?;Uxd4$UL^ znUh-bGw$atG+W+&b{?wrylFCfc4~OLVQK%HZQia0Z&UB8!JEXk6lc%$P}4(A4>diW z{kHJWwbaa0J_UZ2noh>4>5;VL*Th42${v{C<&9sJG_z6DZ^aF7x9#_g^Yf8Lg13p6 zkI=uT%%4hgs@w}TYe{kDp|q=W2da8$*=MIdDCQ&IT?QGMl&Sz}xKgIr7#B)0TR0?>9R6DWIlD(xHASPpWvKysa73#o=uncpK7r zWPRV$R8tMj0lZoHTfP_4^vPdeYy9*`Yo$2XvS<4Bym1vX zOIq?XPA%n;;#^BD`CZ<)ikcZR#0_tK_BrBTHg6Nb+r3>!x$h~@h&^buX6-qB&kkZ$ zQ&*agwARm?W*5vH>gU?f&bd~jUEVaSGI$Jc|99JAU2NVafwy_Hs_>?%rJ1ngk>Z-V zsXTt(QdQC9$$J{{yh$Beal>1ehkt#i&D&(~He_Q)-ZYbJWKZMpmiLt6c~f?1czgCU zmpXr6W(s)g&~vo=o@(-IUV%`L^?9lJ<}yFmGEU8X>3NefXevr;*@>j3wX7?xWn9l& zWJSZX7U3f^9v8^D`pr}R*B0--up9@f?K##NHmbg1!2`D5aG-eRhh)*9aa zd)>p%|F540-qt=Iz*}fP(&vSFG#>h?rL~Mpx_k=!YG`KqW3;9RdZ?wk^yqoxDk-xk zZg{(W&A*?t@3-mTZDsE<`u8;S6B_FG?4{=08d@vu{T9u>Wm3%pHvCV)51uU{mO6t{ZL%$}5= zacVzrbV{0?)1!&gBWWq8x@)lZMv%8YV@OJ0cZ#e(H^BnNjc6k-vq@RCCo;5wwPc5z0 zc%-$V@$xBfr_8{()D`2BW=?AQ8P7xedE-v0rB3LvwBhZXlRvcHzTf78xAreo;Z4J{ z)KAP>DQ6z>`1_4rHS(_gym1$i+VD2-{Lk-c^EMB>Z5}c%+k2|%t?^Gmvx{c;6w2GM z-jDULd7BU3Zojz-Z{&npn!Ls%t(D?=(F>K`BUYQYPq~|!ZFpQiuH=?d5i4O@V4uxdcJ1!b|ZKjFnoNr z_mn3>E%nUPlsuYvsyr@ltghJ+{h>N0zcc|U?(!x%xeJYl{*pGly*zK;7Mr();H~v- z8F{m6W&VA$dgbx+RGipY679tPRpBT7lyYkzg^bGzTcLEw`bQ>;VpDWo*oMOek`;Hmp5tGQarR` zD!-;5dJ5(3^jXinqr%$*yX<__PmVt3$ca5xbiMwt5(}#@o0RQ(OEtWD<#BPVS0{Pb zEflxk?6YxV1-G5byd7KNZT9#Yyk+>dvFeqUDo?7@DW3wbfzZs{5p`&uy#0FKxC?4d ziE*oQ^PH}3{hpoZj-CANZD~2~sCovw)R14+pkRR>#n8VvC72c*iSP*Y{PgOH{_Tc9&G#fKhr^>^e)XYy! zPocbhX7;sH9o|l@@Yd_fsqTBK`Tmd|?v8OzfN{xBkEFR)(p)Qf{Je3MCL8_Kp=;@v zH1l)9j8nV3F*BV~oE~cWOWJ(@Ht{KpkLpF{ zba~6Ey5a4;<96T2;qCMaZ_jP0!dtU26EnALyv0l=W^MdVNnK1f&sxLV-_IC(q{G`8 z72ei%o94c!o}4v*ty-_CNA`^8tC`|a!sZ^L^{&-R}3 z)0g+en4f#6rbp8Bmrp_3g%p=Er;4*f>X`g`-Xbd+-uC$Ux(*I+=Tvy>IzJL${dP>^dZ}b1_2>f;Dw|+n5!U}Ia`_G7fPuUN()lbXs zMMxgTsq@H__I{H(42^4c9hy_mTWDs>Z+QDm$IrRH|8`M@w`Z0l;LWOAlPk$D#Z#RE z^Ayh;t4lkLiA(tl<*nz(7XRCMPcN?UcE__h@)k2a&01-~nzhCArs*n`hm$Fkx5-~# zxyj+}k_vA#2hEIsPo*j4@!d$OoaIwsHfm`{jEDNUR->6;(#&K1@aC3R#?PCyiWON! zikCd*Jw0ld9`3IfF0Jr3<)$2YW1o$n-A}!yE6;>m&Qz19NDq7V^Cor04AdUK;qAK* zZ{*(xT>s|ZuWyB)=P>xijKJ~mq$V0UUOt9K?l4tvdf+r?>hT!ho_K6N{(@GSx7OgT z&9EEtdr(tNEL9%PO3xct<oxE$j4KzFSVqo2Ns=+n^2Ke$akT+k>~eN6mKMQ>k;-3iW9G z%%8XR^HyZHscK30aG|{Y?)&TU7iP-$TLVc;t zdGe0O^x*O)bsJMf%E>tMQ_~ZpUEX4{dDa@ZlOrW0$e;qAoU|GLxWtuuIA)n=~$J(YIBJDFPYYvRmf?WHEeTr1V0 z$In}6bxlQ9PvvJG=BM`arpZ8$M;qRbY1jHZo3|^$+oA~>dGpk*$wuf}>GT#UoWlaEwV$yTf38XTV?as1-y;8r-^xsnUDW|V`dF|^f#I=l(&-)`q2k% z-nxRfPVMLE-_w}MOKYWg=&3P}M!URmRcQ54f04EHyhV0p?zca`@a!8lZ{5J#rpW=k zX{O5W95nuVd0gJ|=tOgZ8V`F5ov_OrGf=ZiXji4RhPOd4yz^k2x9;HW_WJ{PlX_s! z)a;*{9%_21>5(-3)RN!jO>%0og|1~zYAH`lT+bV;Wa!H9_MyG!zTM`n2Y6fCalZaN zWe=g6d!Y`^lgjV%rpYXQ;-t0G|6`K;oD8+(*Th}kG^;{ArO6oH{=VdoV{P7gg133o z3*s%b|4@G@3E6lH?W#z>Gy#`4I;kVOHN5TJ=|lV3y!8Ta*F2PfH+~YSWBS+R(RkPc z{Vs3(Dpyg{Lro90t*T-#ezJI$4y!G$0AlrMI@N<#; z_v<(lNxQsB&SsChyoK&adDe!v!xpc*#=hTrgSXZSLHBfEqlnuTdL~p zBvgm?5YL;`q1B1u?ZKmOY-RJ-2fS@~BqMKDt*UHDeKtZo8*laMTr+LS5$z=dD?Gr0KIz z-k!VUrExZI{lVMIy%zf4)6jQZGXV_`l1I}Id!Xi8KW}tO8A3hMj`EKCdE+kFi4>=Y zy5u+S=~vF*eXz~j0Pyz6{KULjJ+!=!QvZxoGf$`{A=J#H=Z&jEwX`ECo+>}{_<57& z!wizP)*9Xp-!S2?Hg5yL+wGfc@Fvx2_PC$7X5StA_wz<)q&B=w`{$~k*}M$`Z!`NY z^1r8%J+$0!%wv5ns(R?x=+K;g-a@m5`ZYV!cr!&ZbHBZG`R32sybS?w zL!Pd}n=*frd#PTv{P$bEy2_)jpSOe+4R3euv8VI@>xY83_5&6d|DH;F)Z9zb+R*q8 z-jYwi@b>Qgzxr$Yej5hfwk*xaTaj#pvZU!P+q5;AL%Yq}U+Dezk&#=RKTm!Qc)R1d zI`fvt32Jg?<1G(1G~LG3EtIz{<0s!{-*3ag+x3H&xbG<^p!t51{?N&2@`vWs_~Uuw zuBoN!8jrM=Q=+C{io3kUbi#_3e)IQ3KJ!1$@4t-zZ-Z7;;myj^)W0r}_7cyVl)2I3 zE^j$?Xn5P>Ti@zq-*4A~w{|a8;Z4JTDo^O%HFc$&Tw4rptSEJrcf1(hN;@^Y{e4N_ zyKUY^g11M9Ep^{hX#&!X%{aA`Cy%&PSBm?2<0@))&N#K?VVs(4sp*ll#;@m1lZ~BN ze#2Y;_bkUhpjG~PT%*9-eYXVg##+>Qe2QaoYS#LBCm zKY&{1Z8UhB|4IVhGW1sW?+KfiH%?o_o}M>NHdeQ^;jPmpXTH;ZPsf0_5!Wtr-_yvt zK`f=J6`E)J;LSSms*~6A#%!VUEUh)Xed~AcnQZem7QA)1y$WxvD{1TPZrQ06Pn9Rb z9IE!_^2W}oS&?yS?v8Pd4$UvsVw~FLja|{nIJL4v!`q;?zdOz5?K<$*YFkF$*aLgc zL+4SedD%I|o$K@@jT^rs$ zv(t@7+PqB!Z#~D=;EkQ9(mCz7g)vS3#^FuVZOr6d-eR(O)*9Y6ZTrwZHgA)_+e_=} zz#DrGopz|7wW!Odz=%e(I<;nPy*w^&d30XNX?Xk2BOAYL^EMg0t!ul&|DM)sN_p4K z#v8lJiRXlOS+pJ%0=xN_?Q^DJmdlU2K>8FJsGDEFdo5~Y^ zzoqIVG*9Tn{Je!$_xKHOuRSw-7n`?f;H__moAvK$=uSPowXm0mN7HBVylJ}1V;cGK zX6?xE_NkW^y=wC|9lX6dC4e_h!>W}lKdhTd7sDH0DC*E%aIK`JocZx)RW!Vv^28sv z+Pqy4-Znl^g||Fd;*_jDYqA=;mVRp1EuVt)RmQ2Afm-89HC5)()OC52RxwV^ol3go zH@y9(=lgzQ^ELy#-PLKO|2++@EA^8{T+`ngu4ncsiEN!ge|@`T2vzmG5FVGmp@`SrXp z8>>i}t+?Us_5IrIVe@tac$@ohM&7JiRo}#zPrW98jK{s-8oAr}osznkY@W4-w^J|q z>xXULW`VciU2pNfr=I*(8O!jwsG3Kr8=9Yf66NwHRSb=@qGqk+4~?^yyVF#>NnMqy z8{XbIs7-g9x7pyW`>Yzgv2*UI*>!%rHQHoITCTf(legcsA8GS82fV%fXbs*{O;6%h zisw<6wQ`+&(2X6lQ_a7ZR48x#dK|UR=4~!`yQ{~oY2VYB=@q%Qd&Y3zzDU}mE@7)QXa;+R?_rS)6Y1y%Ntiwa~0znU0Q2+`%}MLJJ`J4 z2;N%VSc5nAusv!2{Z`~orPB=ULCTYjH%)cx-ZQ-I{lD)UW%ITWygm730^X#ZYI^2c zjX#wqp107>B|qcV308eULMPzz#tf{^IJM+qyrd0peGfj^`FReDz}x-(R{7sk*4>_T zJa5f14L@%QcVT#YaMJii_WiaPye(NAz#DmHpPG2gCztuD*#ouYVLUBw%&gH;w_Hn4 zjCOfrm6&s}x-z`oaNTA1*}N?QZ#O(sgE#g@ZPluI^XIWPCTIB+LNoAmG;1XfglZ}pPD@~KQ-4%+RvMmO^QoZ z=%=Qi3{%q+qxHPSR4H|CcpLezpY*kPTL#_+F0aB{%&w)iQe3mgn6*;=@+nAP-Jasb z@K&0b;qBc6H_o?tTMph@zfgs@Mw|8y-eOLqP~L`p{IHWNygjhX&PN@1^f5#H$?MPOR{@?bRy0k!SWM#p$8e`03G1m44O=)qdVWt4My$jzT*R z_3L?)b|J;B-5K7_+V#+34sRz_c-uU3wfmlON32CnkEH3Rmi(b{O-{*^jW=ea4(*)& zRC%PFE^n+z%?y&JpSt8XysaAhi^UFaCs%lT_>LO9Npd(OKt>1q;wZhvCW7qiK(>x|4ohEB(JoJZZ_kPn<)ObQWiScXt zae33M(s-=S4R7yXzY;&5{4L&2tMJx;Z9%+c=)dJ&wf0pruBqkn7FwlQ{4Q@fb#8dO zX}6p3*PY+u?eq$7t=p`1?>C+o?@Y!uJ##JnQe5&#bv1dIliKA?%EmY~vq_qrb1gMJ zQXcvx5A)FP^2V=9*)(x0kKygpbIyGd4{G`UXPr^u?V0gac+=F9JlwV9k>XbWHSaOz z;ac{tI(V=em`#+ zrroj&u8rxCd1ADmH%(^cE(~vDzBizk!`oRE-e$F1=f0go zylJ{+=b?VCjoCHxNZREsr;3KRhhBW`4-RiLrii?Q2gR?f&_Y^D4aEIjstB zQs>ILHSaym50R2;#q`FtdfvE-T}g3zBptI>$|=Ru@+Ng&iW}Yzop$CYocHwn3U8Ah ztiqer`Sujo^VaOUPRpBT8iu#NFaGc69o{ae@YeUryZ!H}#IvVv)%%Tks6+E;{LC-? z%CUS3bV`|}4D@UA&@X9CEorS3&yP2DThfNNA5LC3&*AOD3UAwH)Zk6pg(e<)3ZZ^Y z5A-m<%NxJSRgI=Y`^=9wPNvkA;q9-_eQ1Nj+eH=L9@vnWx6o;X`ngBy&^+Z+UUDSR5!d0zV+YkpGUg5!rPo~_xRsaX&0e!ssGSCsr=b^OI5wdJlS~5 z(4pb&u;W*{|3B-J3UAlUZZh69+-f}8c++Iga8gyj<4u*PP~MiEzxR)w_w>>VZ|xpS zz*{4ng4BZ)&&C@WkTOegt}RmgdDC=D54E)mbHCkm({8CHfw%EbRN;;1qM1ynUz0O$Pvh{G_w)+oZS=H5&a-)I z1KxV}xi9T|%HISGWkxf3O@2+Dc-}NsQh7K@PC&{}kIS286)S3VsiJvL_qlJIbHB9( zZ?D{#m^V#td3&UJvTiosG?`f?RLA_UWvQ0So8*j)8{Sr*x8MKS_uJ**?Y=Dq@y1V_ zW?$0DN_iNkW-Z43ywS-F8ZE7r;>@Y>(67;c-ZT{@k99_dw>huQKh5T?9e7)L_5Eqz zQ_Tb#!Baj3Yhp=$)f}2L)E_!MKX0MgJbuI5!?$g<{=MA;k8Tg%CN0Ux8=0pLogUXx zd-l~rMW~-S{r4N4&8}VEI5+CZof_V@{{5=f_WjlYybXS~3U8szME2RjPmy`_yfynY zT;5X6*_w>u?W(26wX%7;0=%^u^nm_7Wq#^Z{cG}Q>S{dYQ_wgyUD4BMIrHO5#(^B$(AhngO0dNf*E%Q&@^)4kugidxFRxP~!}hdHTZ*6MlF z?3x}+8{R&A>a(xfymbO^t6#{-8>i8X+T~5Nio;M9+^iWIHH6CfLCeHlD@K)-|@YeUQUt4bTb|rWlure`k z+^J?V_44R>({$L#yDpTsRZoB99-Fr=;O(-liFuPwr(P!_)z$MRWsr8-Xz@aMJNm?z z54Cyg3f>+a{-FLnZT88N1U+wD)r>kDZyCP36CNz)6FL|=@7P_O7-|#lM>w?#8-nxUgaj)gbn>2w8dnuSVsYA_d?>9P|UFXM}H5tR( z_>mL;X7kn)ygh$M9eA_)tjb%cCzQ{$yoFW|-Cc~oP~P_b;Qk$L-g<$zyWR-k&HDZ1 zs+`cnDaD)!b5h6T*YhS-WOYgBy;hUom9-DiQ#+g0Fg>DY(e_murm zvj>glTI!g!%tIZTvwRB7K+SB7Q`3{Tj;ZVNrsF`M!e-EzZot)>&nqsbHM$;O-1p(k#5`_Wz>m}&FY2fR&f zv%$UJJULq?9C{+m8MBsosF{e zKTV;UQC5~z$(;IX34{NsFrGl#-+M0Z*;N>;~H(PHTT<*PtDk9-*5fE zTaSBc@Wz~ZYl*#>x>8Q}ev_Or)f-{0%Ue!6HM|{g<*`@Vyj>06UTF77w)eCVCK=O% z%UevgMqcakmQ%Nex3~S~;d^b~`h&OoCfDFCr_Vym_ko(dvj=Kvt;XZ>##Pi(m6&+m z`PrMEx5%#E;~LGilGfy*hbPON^hj}+H_6F3wWdnR zZ+LsS_4jwRc^d-WUbu2&w)d1hG@~w`0?$#}g~lVTZG^bXTTEA;X&BymefpR?ZQh20 zx4UNs@aC!2^4%{zlGf~(Yc+m)q_vFu?>A;^1kKJv`ze&S<#X;l)8=g$c$@i10B@l? z&D$S;3VCH|U=Gl?C-~PSk-S`I>%0J)o8t^u_`(yF%Y0SBM`fs^2^YGO7ek*cf zh4OaF8$TIo-*3agTiZDac*}!_(43(sSUv@5730+Gig9Xs^46NVh4Oaj5BB`H&D#j@ z_Q+%Z(Y(d*pgHT9{HfL!%G+fNE`84C?OO1*qSvNu?`i0?>8DOL)zCcUQwUwvXnxHk z^W)8`ZoYpz`;JwkY~DtKxAF5c^44gRXFvSCBWilG@g`+s<{~vuT^n zPgC_-uRMC*Gz@4w_1!6{VjbelSj`RPp26*Ny~ND zhPMZf{{j9%qw?>+jRkKjo~pqc-*;+h^7K%X4aTKBjBEU%p7P|SRg6=!D@kj1%A7Hp zJunaBE^l;7ni(W*tu?%je`BQc=Pj=TZ_E2Xp?^;~NouK|n0Vg#%cqccm6!>nTI=$b zDzlZx@V05?+nt{uG7h|rTN=Qdl|R?t|IfQt`4sZ5!u)x#t;ty^Zzo^z!Hezh-^PQt zp3mjTn`RoC6A8`3oRZGQTOOTA9WowMw@}{ZJ@epjo3{zz?S(;`-S;$Rcbt->t#`lX zJxRZmGc@k<#wygB%#vS~rYGNH^EMH@J$6$S-aI*L zve7JS%cqd{C#zo5;J*C4CG61f_J>x(ocDARcw716cE}s2EzPqS-ZY&^9j1yG%G*BU z_xg){zfA^jlZHL%zNgat85-xrq_y-EsmrIpilH59{Gpya`1JFZu%fx&j@xbhf7!fE z0dIqEslppiAz?oa^%lFlan~`rUK5k*`gx1lskPSd_RUqBceZ((3f|hjQiC@uXG1X~ z^(IZ8{``1jl{~a&cdT0|Z{Ij|%%^SMrh&IjBeuBjDR)H8T}WE<>mMoVj@cxX=Mq4x8}`AZdn}^o(7Fpf!_R}x#`-;um_26y(>s5G*tXuWXZOl{PPNh#SJ=D^U zLh~?Z9@@`a!it8sKmG0MLu}q=fVVNDpK{;RggsPcFOU4v=}Wcr_nVZVS>lEA)?=q} z2id&M1aI9}C*Upgtn)spbNZfLlXI81M%zWIDHY1wygODMYx8ykczeFp)7!`W7J51^ zZ=o5Q#a}3IyKiZ|&E{Z8mtD zyDoq?W!;)@nPy6oN9sX}OL_FXg=Us2)+_GvmQyE&w=c~d=KOu_t@3*<&t^LG|yfx$GdHVYMjjNhb7s}gDMs;6n^EMB>ZMi2S zZ_PM)_FP}yiu|+|%GYDuYFhAqe^hlb1KW}tuv{bPaH{ZYgVeQ^e+4tK5@OI~vD!fS(lWIwQN`A(< zmT`^dT58Q&|NSOSAhZfQ;aY0eqCZA6zn?cv6=tK~iW}ZGcKEjQ=RRAe6IOj`QS z##_veLOWzFX)WW_T$_zIW>9Lw+k5w0{YU#fT?F2)xV{E&(k_%ew485|eofDQ-ZU9P zcf_33QXY**v$jy)CjR-5eQe$qgSSl&)!o@p4~ zZhH50=jXpI1#e?-NX(n3pB5fM_n3#@jYnw&8c)2!9=rpe~nk>PFqx^~Xb<5~vZ zx;&bgH%~u1cuO*YP%8Aig`S<&iQ#SQ-+sHB{hlrdZ_jmqp?Tla(D~5cIQJX7%0uVJ zo7Ju1ZSA+;w||AV2X@){sDq9^=E#XXR&>4ou+pdPo;g);BevFW>9u@7rMT5Xi$Cu; z`=OTV>TzQ>YAJKx@z5T!;l?T&UD~z5?K`_&eM$wloyxo&Tj6cxrYgK?CUyz_2rac{ zEj?VzeMuhr>5(*Za&7q(_*E$b<1v1&rDlF=$-`aH&p5Tqo0OT^G@4aPISp@LnegpC z4sXX*c)R_o7yZwvrZ?V)nm#q2(3~1S_u{|b=%i-nQe4vsJ=9Wusg~qt+~tj5rDheW zD=A*eV|ZKqlL;Slcsstr+pGmOc=O~h?5*TXQfWVLsj9@}Ni}(ww^W&}-5K8YxU|O| z4sRz^cpLI$LA-_TG1PDEz2;{j8*iGfG@j6IWBe{}G1;WGrCl4|-hJ17mpHtgSmEum zelNN2X`@X{+G9*S8*ecc>$SER-b%YOysdlN>RAqNCslacw5SSi(qy(L4*k3}%T6_Y z7RuZAP96U^rU_dT`pRQ+DaQ@5&LlQYy4(?jvR#dI4w8T#XSV>XXAy#4W#%T93aw^J*;^eud!2Z+n#){OXispV>d+Mai9Ntc^@b<{yt=ZmF?kH4SeFoi!p|vD`W8QC4 zw=wawym>k?ydC%LH7_~5ol)U!#fpM><4r;>^`AGB(>isR(PBIQbyjS$*_NF_AkZhiP2Ji_MGj0lQJ{TD%=GxUEa7;I#X$>6DdFa)RLcZ?nTcVS23GX z8{Xz#dEv_rZ|78a+ww{c-jucK@;!e{Z!T{!+45c+LxP^S$XyuTHr}-OY=^gVE4;0~ z_Er6R8q-hcCzQ3gR?_rKnx|Gi1!j|0F&^sY+Is0id3)@bcfHNw?Ys(aH{M=_H=awq zrWDhIW^LO0O;eGR4E4vXEtI##v+iH*@OFNMw`;al;f=e?TgOa3rnmAb6j{B@Y=xTM0{l)Gy1mZ5H4 z-bc=jTI&BlnK#adTAGzIZg?BB`-AUvc)PU1Td&Kv`QKC4;$14m**`V?)RISvYu3_3 z&Hnwo(a8*6GG}w&aoW&@bhohg!-h#a-T5k*h+r)s^Ax*9+I4 zYxC9~ysdk<3U5+f=9jdzR*J{u(acTC@AAeDxr%W~(;uUmU&=#|M!UReR!P;Rcqymh z?Zl0T{M_cP19-c$Yis>`%37M9B~Pk6%+DWwM=jMVp8_*T&QK5iF`o$Tf|~g?J-fUy z8?`2zr@G;7|8<|=&*tq4@V07J4c?f)URs(%=xMTl_kNSAaFs@D@@PDoS{jebn`Tu8 zkKt|e#CKn5^VSi(&D|Klo928o^jY)Qzj@Sk@3%av;uUujh@~G}SGSx!+bC zeF^@8R{8&Dbp~&J=T+g2dAJM4n_av28+Wul?OM+pCy+rK-tJue@+a*3?Mm>r?eQAC zh1PABlk)Qxx@(O;W->8r^}NMYDNVrecGxj(@fYmN_gfe6_E7IO?t7}4UMi2~4iEM4 z*Id-hQ$7VosHNS}LoNAPom%oUt~q^84=!)gDt0J&qu*fV zn`TO!YQ1zZ_Zz1{UF5D^-g2sLc)Q?&^Y8qFyI`ME+|L^`P)ph9v9#gst6TndsC~cn0&kBEygdFr&CpvCez@Pz9-5ao?pmX> z@utaac?@qmU;D1}ZQiZ|Z#OTm!kd++CL7B8wq_pIWlm~(Buzgx{f)z0=qH6cqUP?X zi{Y)*iQ#S4F7G;BsYC1fd5g&$vzEI^mB-~RRc3Zu(uTLE zHlFPKdGbEst^d$=?t5zG3A&eZ+DlLmn*RN~X)@HyqnR*0E^ncoGaI#)$MAOX8)uzq z-*0`v+qRn%^JevsoRd`fo0qrH3B>cJtY~<<<(083Y~K2Tw?|&C!5ev|R`yUA-ZXt? z<4uz})nr(UIy8Twye;4R&fRR@t_E*Qhqq7ro`z0C@@u{m=Gsudp107UlsV?kDKck%yji<2yj?Wpv^h3!1HjwW zul+~!mWr8Fd3b`Uau&l|sawO_>bL##7Mr($;O+I19UAjJ<>d3w-P;jL8B@V0s4zkgx#HVC}ke@C5p(@c!hOQp-FkSZJV^74B+47_c+tfT*aV=Zd#s2R1M zw`Sk9%Ue$8_2&KdmDzu@@3(8f+sg4Zc+07`y8M3!?nTcVS2c^~XQWWxF7CJW1e>?v z;BCs?4ayrQmd7cVPa*FrQs+h1EtI!APuuI0Hg6-qTc38F{O_qWv6#5^|3T_<*Tvs& z8fLgtjgFbB)K8(j?fdNQH`u&g3*K5!&XG510x?rfwKkr&&`zXjFixE+f1$jspECCM zHg6-r+lKq9@Fsl{r3pxW#<^CC(?d-^PlR#$HQMD(%Eq`xYpT;D<)lZWr967xqztT1 z591|me!bB1&gjJ$D2k$vXMo7AU!zctz}q|QV8ba_ja+3LjbcJ#ZxKhfrG40s#)U`F1oTDdZl zN3FEH<#d8crt0#R$2?0p4R3u<_|i!>Z)3q*mn*N#_MV1LIMknNCQ@COH_0i*Q*{`V zN7Gw2-ZYu5&JAzB+Vu7>+Pqx{-nPuh$eUFw$tSVMoGx!M)pSsTC-eRhhDjMGY z@~c+kZQjO%x5?eQWP4A!qej-!3|-zDc^B+W+7Ug}dfvDygEqYV>Cw;4w|ScY-ulc= z%v*r0AS>q|4qv7q}>%MWj&D%uqwsm7--ty*0x%Zk(y1X$n zb>2J5JHN|YPP;R_eQS@?_p*7L1m5oL*)_lSG^c&lWJz;^dlkbY`S#XLOhfts}n zL;l*6TC-N;k(kk}l{_wQl2eLvr_`3;+;6Q% z?Q@pR+Z6CN@`*a|md8}Bz1O@0v+)*Mr4jt`yeW5Kc-!>7&L6jVn+o2#^y%ilr(~F# zbtPu#r{-Eor}8t8r1iW>8Khl=#+hHUR`N@6KX1}1O(*n}@)+J0_M78;|27T0UB0ji zZ_K07>{+9woLnn;LgUQI&q(@Y@?eCA2j1M|4Nv0|uZX6jNN!`p@dpLoUQ?RxO`)YaYH z_cXMJX7ra&ft^bX(9>vI({nc7G?}ew7~YP1`0l4|-e!QeJC^L=?F5%SYmzm8^`Y@= zPEhhlaZTN9ylFC9T^Zg6+|zTF&D%`yHvQSeyjeXIc2b)8&=b#CbM2JWaSk!snta>!RGA-@YZio5C3~=+9yeKcdgk4D^go)4R6c${fqPO zear%HTbI{?H>=N@414xib1iEzkACu8C1qngMoYC~a>nzftZsNa{3i!F|Gx8V@OJkL zHF#q!{(6L(9%_0tnro@0we(0@TC3-cnVBt=4X$O*&^%nL(SF`ootl}YI6Wn8cw4je z2ankA=^XI3d}vSqdn!%Isugt46uDOF-_M(J!c}+8wNmvWC*bETVJC*SJ$8QRT${JK z;BE5C8oVX!r|Rj{E5FMd*`N;XkZUC!nlr|qmN)Lm>dNr8=OyFE*}TmIZ$q{Q@aCzN zD@%FQb$PR7l?-bQUa z!+B2^g16bDuFCH{l{k^&MV{sk-jYwi@OI?aZeD8NZ;Qa&&^vSFtw?@YOT%;8X=^I7 z!x$Y}EA9Q}scv|?@qu2WY~B`wx651g&hLJUnQE%F(tOhLma5x4@}%X>)1l$*6K{Ot z3pQ^{z}x1rO~9MP4EG-D@$;t1#_BOTv{tA;EpMLchPP#J|L@~%-j;&5o7X1bjlW`| zjyZ+A*ZO(mE~ry=#hlcs@-vT~HxEmi-;Ff9wR-P>el~B*z}w6=eVX?@jhT<1H~#De zo?WQcbV!e+HFdd`em!qol|dWce!lPzN87wD2XB2R)ZmSuzEoP;iklj4 zkv;CIft!_5bFEhONW(~#KQxd3dJE0g$bLO;k-IRs-SYRbFI8~csm$B472dYC>+3$J zjmAdg9IEo4jkji;Org9Tw*2G&aCkee!rT3mtMJC_lRDcf3 z;8&@cS))VO(jTKWy}7)xA~P_q(Og^7=6-AQz&ZHmKHj?Dj<4``&;3<+v+`8EA2c2^ zqw&XhLThEa-(otf*IJjim`*%%HoT4KI=PK=znxIwZGOjo?t2>3PqVDu!CTr1lsGiJ zEjwo1V-9a8R(KmZJuz<{{xjc`WSIG>>Cx!4)7Dh1mnZH0=GleeZQ@0T;;+lR^`4$o z;jQyS8F`c5clJXM3J^jXvm!13VkI1TF1JbC!p54E1RgeTH6Nli|6OC35T zKX0MgJbuI56`g+eZily1E4)4NNCMtGJ+w?%niwZ9d1CTQIbGh^2`e%KH9eZO^iVU8 zq&0r2HTvO1q{<(_5&YIi()x(dhCiuoI1DX8IXVrKS8*oOS8(^F}8% zvnjRV?cfK_!5{B^>-R&>tnk+DssZkM%DU9bx>fHqa>AU{n*8)rYu3`^=Z((1HFrVH zx{On6JbvDoL8Cp@4R2q29;$o;PNbDwg7gw~v40)ORcwd9xLe%_QURh>DzDrv*pqP@G{@9=hBg|}f( zC*V!mDf^K;^@?ZXE#`EJ+$sC?^Cor0v!nL-4R8B@b^S7jxAQB!bs8|J@!wNvk1=tV zH=a>U=0;najkgTDFuXn7Z|I>8Zx>W}duHhl-g5l$gJ$J}oxp7Ea1&i!^#g|}IQ2RG(>nqj|Hzo(UE z<-Vtqv)SX>c+0SB!`lJFPVDdSc5#KbVJjMlH_i8@(i}q97SCH~htf_%<1TOPgj$o? z<2Srr+~c#iIlNs`;jPTi?H3TH)=9VMF}ysfXvP?-zUHTE?wDtLDku?YGnbj!jT&#!3%0^W>p5waUk!acVrFyO45*9X$l971#3?S<&z| z`x9+GZ}Zj$ymh}lBX5ztwQyUnJbK>h)nTf-*?0@R-%5VN+ZF$F>N=aZw&3lh*8_Mf z@{^mYmXxP)c#}Gm;^a_6MK<0v)!9i&8{TIA;jvyeZ9GK8*j{@)P}cD_I&?jo40o0?WWar<}KA!rI|?ii{VY`s>pbuynSr#4=32X zwFhq#T3@5T-+0GTb9#(Z)1%Q`D`_bY{gS7A3X+p?iC8H!shPMxY_y6r}^L7Pz8+2C{-ZZtO9vGLjlwa~&`>Of9qLiO`T;8~f zTIz~%jbHLeaptF%{8Bs{Z=n@Svo*Zke)^5)*}QcGZ#^y_?tf2tM(lxF*g!`u8X-Lk9ATPN`L!o(`PH6uTql}5X~X?CsgXsT;GscL28P1B*} zF}(fj*~hJaPyDAxcLs0k?ybU`m8a(WrSYel9&5$qba`XuA~knGovMdyyk+Rb@bf6mc8b9+>GmoFQR8?4!olvuu(>&NmVo* zmMR+F4%z$slWg9)fwxwjul2vDti@e4qvl@xyfypo*uS1PuF9YdZ@ZuUGJarm`Om*~ z2XBvDUxhbvPR$+|m$c-U;><%&D$V@mQ(#1@Qlv-ffpPXjO%Ju6H)hjlRw?-nZ-1M* zm-GJ@dVseFAFje1?*ZndW1yQPP5jn?#|$&-yYO=fyZ zJ2kwWb=Ya2vES34;B95sk^cA8%GvV$Rm>!rvq;UJ%cme!r&FV)JdCHx!<>>Pt1fTs zgc+nbJsxd%d(R*Dy};(J7kHaLDWUubk^GuC zb87tbxV&k)qKBHDP?sti-oAV6kgILpt^#k1HrC*cpA_zbnsuq^3DwL)9hyh-`|met zl_t(^sin1=2}^4=ahEquSDMW9lyVy0PTzMFen3+B*9*PD+oYbOwvYEznjz!Ve%|Qh zR5hAwsWoe3JW?$`Z_+AuqVbgchPSUyAG)7?zx4rcBjyG0#vU}9eNuC+CQc7E^Jp~J z>UrZTYN;YU8m-AAd89b&`gxNwgvLu14R7;?f9gV;x4z(Q@Do*d%ln28-HYUx`jq0# z|>j&Oi_ZjWJr~LkqTGJ;#S=6ChDX?U($xR*Ou_#&nkcab~SkGys!#y(v(8u zQqQ4z^6-~Wp^-aGwQHBRoOW$^JO9T$oxcy*AG|%iB}d*kF@6>pub0-}Z}sXhRb7|2 zRGFF>X$Ze z1Hs$eB{g^}l7S*?`FXP@S@%yz%w+Q8&FaeVHvEjkK5p|i2)qq{whC{2`>3t_HBW(S zsYCNfekmtup@;GEDe$XO6-}IZsJWKa84uM`ekqTiH)f-jvU%c$w~s&j;g@XQ27|X= zgU0&bQ&0YyyfwmF>6G33O>!2wi^vmiIT7aZ^QPGyvr$`X4R1djKKKTkw;|x|_2t_k zZyIu3-mK}hoL#-*nq6pm^YbQEVVv4i(eSq5gP;7g&D&7$_P`5OcvI%DdLl7vJ$p%B z%jK=e&O>(^WzH}S!`t<3e~5n|u>AXP!@%2uq1Wa2o@VH|DpxTxk$P}>ld3dh zoV(7(TZUa5-VR&b?o;;tb`5wNv9bzpe50u~J z!P}PG3gRt~Pp)R#lBe-`OEs}Vd3)%`kFT`8x7uu+_|0e#$R*s#weeH^i?|P zp{7UDG5*lCQV;RGNtvw<4R1rf_{1Qaw=v+YTdN6a?>FhQAjPG4**w~srGyQAjx z^5czFO4{(&`Nw|56X7hF(c)NRT z4c;_6%EOb2b16?Y-lS}KOn^CK>Sp6jswl-v-J1Jt!Tf7Zv3VN@-e$L%nD#y8lqAh} zpIY+A#F?Mk<&9ruMaHQ)dB(XmR5K5?l!qQ_`u)6dRj6i_k~X~EbKm1Xw|N^6-lk4S zz#AuT_0aOprt(PnL*p)Qnhcs4@$;6j6T{mtclzXRHg6NaTi<&Ecx%RKOY_Xe zo0Lt8rfG?w=`*|SZoj9K z!P|oSGxEmoEv>piTuFHtr`FWsv?YK1{l>1SrHb@WYy6T&iZg#aZ_MV=hPTl_TYiSk z+Z6CNz2oHOdQUYhNvFVgJa5d_$U2@k#qgGD7cqGX zl03!mCRHyoUMOz|wt9H3eZS2BZ{r@R!CM|@t(mr_uErzP%EntvMX8g#Cyp&wkOq-)4ii`Msv?cu&*bZxRDioTne7rTl*0q*WO@F}(fc#vUDP-sXU} z;q$BT7Slt_+6;RPdRdsinER~}rs3y}Gl|rOx1HYg@4hx~bHQ7O%~g0~-Ov|-en~Tb z-dgG*+nvUL)t#AB?SeU}W4LvBW9C$ym+~0i+CM$a`T1}2z}t&`rswyb#!N{w2Pr?} z)XbTUHz^x4=dGn4q`1plP8}NFULG{s`S(8NgSWK{Yw#xRLW)a!3H3{PQpKB>H_Ze% z?QFa;gQjB1V|Y8{uYWqweoq&Gw;4~>;7zKPp`Xm(Yq1BZt`z55>TJ9*L%pcOag13JCuTT4)hJJ6L@pF&#Q%m!qhq`4Q~@J z+4DU6ep?LQZXYzmeNUx3(^ISF_h!r~?N{TG)=F{aq%NO=v?DsHr7DcmFKNloxTKj= z@-Xi5#$9W&(a+4(r96hWuZ;V}mu=pbfVV|ARpE_!wpZ=vP1?0Ic_}XCDKhTzmPhr{ z1PpI;?)}YSHg8M8+o%_7@D?-4n6-J#ggMKnz=%eN?kLo+=|SUR-FV)Z%~Rd*Hssf< z$Jx9s18*IN&2-;WPtKa}rn0vrd17j1<1J<~^;#Rxo3eAm+c^iG@BIJM<>2k9l~s7t zyy=pMacZe&jfZRLmo)v<^p{V85y{E8q@_IcOPDl^sf2@>_AkTl@e0 z$-b5Q?SWl(KI&&jA9Liy9xJ+De^}{`d3QcdNr!sHHR(M-6;)b-(DfiOI+T~5^HsOiX{3b%`AtvtUEhe+I z*6?=dd++_N!`pEc-X^Uoh&P@ZwbgTBC!*&~(;@%Zh{j)}r%>L0wtJ789Nvzv@OIVf z8F?c!)Ok#j`Ke=Sxx9sDrk`5V75&tjwe)D#7RuYp_fOu{;q8P9Z_kdJmG(X5w~(4? zaC+3tLrqVpW}fmXFrvv8x+CVN*3{DY>7izRmp3}8xk}RXQSE{x0p_>wT8FHel+kBhqsd}yj^!)72c#BNpY)g&F}eBUD4p>ys+qYLG2=x6sh+Trb#3UA%kRpCvui%?Iaom4j7LU-K={z7^C-RlFNadUnE~Zeu2&jkgTDHoSFz``8T*Z)a3^o8Eq|`<}+^E<>L! zjA`m>JenRFhqqKal6GAvZ%1_e-181^XI6N-dP)`Eq|RgFjrMdTvkF^xxaB8Bqy z?-{MX?(lY2g|}_@7sQ)pGDUid=Pibxn6-uScE{`$r#ZZxUEys*$9e91DxEujZ9=V? z2L04hE#{Q`nz-bN=ZzgoyNHRibE%dTXWeYPu{yI!6-$2e{oDJ#^Us|f-p;A;cH8s- z-dI)U0%`=0VHrS{Zn`86i}p_-?_wTx5K zLoIn2_wy!YV+LyaIVHv=zs94<$^6tVZ>%V3W>9LwTaOdIvcJRI`4!%}-%y1&<`34y_f+HvRtl zP16apNq+8*TJq-+cX^Ac!d+08avI*ocYgP`o%`*=3U8acFL2*eX_AafI%Y3=m;r@Qe#TDM>Y^uVWm8T|eMfyWeQ1bhE3*C`a zB{WYSem`%l7`Y3>+uJW#_Y3EKyQIR~(5r5A-&6KL&3-(b2k|N8VJ-GREqSCk*Xntb zI%GCU$E=ldGA`|uaX)XG4mF-qSBAI0w7+SV!`r15-a0G@;Eg+NcCF`a``9U|BaI1n zXL#Fxw`IqIxAkxS{rXn;c@A4P?*OiW{(YH*$KUeI5VdR{GbNw~xaS9K`FRDnGHgx^$%(u(+JI}+P5y~-ZUL*Jk~A@Z_l0non36+E(34V7gga+ z`CwN))0nl=9yK{N9?qwH3eqYm&I~c#G7oD>IqBDE=5%?}WXRw#ynS!*QL}B{+JLuf zp02`MhFVqG(@c+(DY9<1`z=+6F}vm;CDcr~P~N`#&ep?h-r9n<_5&8>_nvB|!4nMi zbFHMMoPOSk;@92v^kW~KlLObww^K(0R3;6YE!qW*o&CGs3ZwU!EyxrHf4q+c@)mp69E4Ak@}wc)L2tFIqn z-)~odx7IH;8E?wH*L;2Ad5b)qnjNOf@8>PmJWJgg-j4X;5qI0Xbp&sZ3|r#9r>T0# zJ5S7s_<55uOYyvS8k67UEvH==-df#^e_m#%@_X6|ye+?_25)??sW}10sp$#T%p+;$ zq=#`y`+4IkW~L7H(4R`PA8O`zd6S%6C27xE!`r7m_{GQU`>iv0TkuL1-aI*Lz5-I8 zQk=D7H1jhLjK_}-kyNBJo+bZ)TwIa z#~Zt)PSts6p8R;Tc4v5dZ1gub*}QcHZ*8~LnK#XSZtbh;eW=O9lPaHrlv#>1Lukc3 zavW|*6dx=Lw@(0rh2MfXll8< zNvk5eGQ3UP^8EkUymbd}t5!DU7r#?d$4(Mz!`u2xwvDxU z>jBQHoX1gI|t$iMwfs8 zttWUJKCTLH?3}$(OTB44Tr2sdIM-53ekq=9@=`W-LLKT4U8~9O^2RFEJa@)fg_@p{ zHoU$3<%fpb_ggRU*5|G&yfKf|13k^C%csEIQ8(j`q@MM>Ng0%J!&}#hL+}Gt%lF$= z;O*76H@WXACogGD4_vG97wK_%OI5wmc3miM`}Tc)vwgqy25pHF)ED&GV8bL%-zF#AEVsEj8RG*@Xf^GJS; zM_MbzrPFbFOI6*fXx`H!o*wr%o43B;ZAym~?t97|Svjlzo`c52oYZ-&)#Ti9za^i5 z;q4F8e*d7&TR-sDduk2dQnA52ji8fHyXq$~kDY2hP4u|;TOM7NavI)7e))61w|To7 zyuI>372bH?QfvB@{ESm`GBNozYw5|x8#||FX2vBg`5D(}=9m0_-lPoDxk&L+b;H}f zyFPr4&0Bx)_GG7<{qL!?iOHHK&;$Z|o$M*6c{~ z6w2GSa}RL-e%2uHHmvK);@{IEPeH2HT)auU(8OKdBr-MGJRb9&-v8s%4zcgI!Qid^ ztS08ov-g_c2mUAX7Be@`TEpAIN8XQr0JZ%8vxb1TEss{=Er$P?wcJapJj_YGgSX@p zFuZl$^`Kkq`)w$AyQ9Y~&HJ8`pHQ7`+IfEpLU*C@mv_qlYF1_N7~Z~e?;d;GybS|y zGw0TUw?;U%Y`pOl8etdgN75uMe%Df!Qrz&i{-*W#2kgu5={4YO(BpOB&C_$u@4Gao ztnuVJ`6RoDnG)9)%G*!3{;_hsz4iBE5jQ~vG^JExXBQe49T^GjOFuZh#6 z=Z&kVr4FSy*HY83iF1+~zn?c|pk`Op9*?=-X7&BzO%>qE_uB~Y_R4}PyfJ^Mmim$6 zsq#yCcslfyPeJ-ByOlJvNt$aVkJJPGnmqJV>v`iUsUqW+HoX1fTVMaE&D*u$ZS#|f zd1KF-NzyOrRC$;Sd1E$7mpU=LO?uBGLu}qgfwxJE8;Cc}^rZcU#`EJXw3AeR zzHZFv@+LXCO46RShPS`BeRz`1+i39C|Ct8jP1<7~@y6jT?@yV_n`WL;S5mx`)9`lO zdp_FM=4}jkYc+6{|2_5O4|>0(J?4>Lle2iWnrtWj^T{dsmfwwi! z*WgX6rHRMzpvfQOiRVpIofDwetWD){dCRF2!`qYXzW)uIw{hTY=8)U{?`f)CH(H)- zyybLKHD8LD>AAecWb>>wyzTYbzb~+P8xP)w+?*qCyl*{y<~nVuuFD&{rIs>;#$$4F zEw!eHLU~(s(4W6x^ELszb$K~Q-Zaywm&fHT?@o9(Qs-UQir9 zQI{uDK7~B0G+NF=dE4u*Z6C0Cn*!d(j=a6bL~NFK)hylJY_BWYHYG}mf8^ixY~^}K1au?lBiT5EWFTc`JSvw52i-ZqU{?SD_1 zpE|TRuBB!kYIZ+P4HA9v2MdAlCGtzDCmH>+0F zyQ`5sdfxac&U;6B)bjI|u%h8@_cx~ZvU!^U-ezC6Mt{F0?5!rBdDkkRLTD9@U*pM} zTjuxkMrWipyv?|I(nOoLnc!{E_yFG64_^mK$M~hSnm9d@cJDXtLR!T*{nYf3J$fX~ zwHnPllHbo8Ge}y>Y{d<4|N7I*BW>Pp0B?QnuEASm4=wjtOde^i6z3cykDfPX&}62k z5wy!2ccjtET^QcJf6&p+&nKS+-nzG2tA9_Wy6jo0p;PSlbPjmC|GpZ$SvgzY$I!SmVJWW3&$W`4^3<0%sjEEV zh4S{^o@ZWQ^EMZ}&A(z@{CgTYC9)B^Hpb7oe%|PmbWA5)%lwR|(k^eI6_tK-zg@6+ z&`O)PdEjmEw2Zv*?WUHdL=QFnj8oI2(b8JR{k&$ME*2HoaHbye$B4FL%DH znD;bvN*VSW#AI6DxI5~+cftHp-9mZW^O?mzwt2e|yxlvaAl^8cM$qL`kj|(PI+S_{ z?Kz$|WkqwpedgGI&9ix12;LTKNWh!)?n`lLl6lA3Px%xW$-8Vz;~f?`clG1-*UB&sR%M`l&g?@+mMCo`@f#s9B z+Pp0XZzG-mBmJ#XP5>TamVyD&E-ZP1;2sQ*v>Wsz=5RZeMKu zH~jO{y~@0uRORjEB^i0M(9V24hwd?^F4s0EZ|sm-n$i_`7$xoJEtGJnt5P1r+lAkr z_@3%>x{J4ytGsP_CL?ds1X@bGIeC+&7fJ>5_<5r%a%x z_f%pyvY*U%Ap4=#^dtGpr@*gLYcfk7=AoASjK}z;wT1FFyIR(X4Rdt%<2VZxfZ?cvRxK;+pO-u8OrJ;yn`{iDj;ZP#ve-&4(bwRDd= zZSF`jd5woT8>I`q-;U|Cv!lb?SykRr}6BCa+UCu>)rr<-artPa>hN}6mA9K;t-)JPzduuR!lAV^evPMm3X)TbOALhii_GKaP189&p3cqv z_AmeZFmT)T=cngad0TPaLwnxS@`*^342>80R%&|l^A_5z$8UH$aM1_x&jEMwc0rZ5 z8}8W6d6VXomNA~YRFR${wVyY3!p^Db@o2-_gNK}n|Gs_~Zx>d1>)z^N|NX|hRMMUv zYQCY8pZO(Es(609rRp$M9zAbTMdi5|-n#wn8~=(2wfy%jFRJpk^M*RSNj=2G`Pre4 z$r-vf8*icAhWbNyq4Dc^i|p3$_VQ7G7~t@Bah10R?rH#U(v-MgYRMlH_w$x_^_b2> z*Xns=<_r@syj|AsSMJ}pyrjz8jcp(Czo)EAt?5njb8VyzjZ@0TTj;rz{D!yg zXI_ARp0(@yw@a(M4WCqpw+wZIzGn-~&q;=^rQgpRosyP1k>bolkEH33(P?>Ow^mn% zx3hMB_bbkOdRdjX*7vml-aLB@Vw>Oh#`DH()S=ySEwy!;m7Rv>&&C@&p=MWBCx*AX zAM5m2hqueCylrX!sQ*1>{zhqQUzN$n^iw{CJTk|0m1-^bm5sLyofzIWUVY3Ghqo)L zyxlUj25%D2?13I?dbn%Gsrj$iq&)Pe(((5jJCQnPoSJJ>X(@kN-lQs}xZ!R64}W#I z!`qcr-WEStgSVIoNQ_BwsaB|8$|J>H-tw*}b&@JB^`DkEcFT$-ZFsw4?XCRt(@k&w z`}$V+c@85w|I`1V5@>1`rj|URapt7flQ2eo51>(JpUX6o<$ms;{$al_j;AGpu@|HKbF zu?u)x_*5O@cC<9!}>M@!*simBH-lPo5xZ&-}wjcVY z{hoFOZ{z!KcHdKF{tA1|=GRc6=Z(A5WTrnxht{H>+Rq!EG2NEd8s0W6TRO<*ts8jj zwrn@yjdP=xW>`K2I;o{qjBETF4|C?B{k$ctXn0$&alkKZ-nxUgXSVK!ylG~b?S9i_ zZZ=Og-ZFer3~v|xY_Ct+y!8NYcMRI1e^2?{f<{Y|=UT}V8V}9c{Je$kNSatS-dLU5 z!=d5rowq$W#OCd4@V4xxI=rzKHFqb)Lp_pTlixFkn%`%ZPl1`KV>*%YNI5ledO~$P zZ>-4OQJ3a#c-#2xE>+z2D*ySyHQ;UT3pIFS9%|OjL$emOrWXCw5{LTxjoGN#RcM@R zW3*J4IbGf~tC&Hl4R13}=(t}MxH4}&!P|skPx{|e>AaLZ)Vy0XYxAz_@|Jf+&8{^b zO`r6*ylJXr@EG2+xBmBkVy?|wKkznv zM@HUq>Nzt%8n)SUJa3^DS-nVIC~x2S)zkQagXLeJ_6KhR#ysu5r=gP!^%ps9sg|F& zmbfGTPD!2QjNJ8`zh3y|BInl&1HjvB>uc~P^|L$0{k%2XPBkP*9+H;pt_^Poer?20 z?Dupaczfiv9C?!_kjH6;*2;DAnZGOGWVj=0dNevMZ=PRM7~a0!^^|woybS_xH(mdX z`<`+aF*{|RJakNNX?ct3B#+53XIkEv+0ur$~^5@@12K%x9xW);7v1) z(C=_U{ZbE7p0vD4)te!%=S`ZG6t{L~c$;+e7iQV_+i>u9*HzE%c~9fnQ=UjezWdel zX7Ll+t>Nvl|L%RZ&D*u$ZQ$ON8Dk zHT@0a?V(F9b$-6(NbuIV<8v+dJ+;1HtvPv~zIzIq3|z%*dFxblv)yl+>elqkd-}rc zL-x1tw^88j*=afQ#-6G3J}FK9`0v>?Rhq$*jkgS)8{Yoo+BPF?-bRDB4I2~i)@&?k zX4Ra$X(p-hZ~}$$)@8<7zqNTA1K#F#c|O~F8oFyvAXMj3t2ucKos7m`C~v>K?xEf` zZ)3sR@L6?u;~P!Qol0?fsPo{5`87J8H)+>WT+@}t!~E3J+Cq8z#M{O{Wb<|%cdHeE5fAB9hZ`XsjSG&KE?LE~@ubDl5 z-cohmJU2P1OSL;IkNNe&uiiE08Jo9p;O*hLb$H{%sI6L+-;1D!TJq-+*VN6%ns^mAkE!q6VV{G0g zfVcJw>+r_!V5y~@N^xneCa&?YF8z{rd6S%+JUy%^Y01x=p=;@9erkIBym3`%S6o}F zXn5QIZJ+!fo41MJ?TyWKcw^_(Qg2e6Yh!e({4Q_IENM;WT&wZt?eX)Ls(NYHhPUth z*WsgV-X?*!jeWMe?`h1$$UJqbeKAi=ewVkH%rTv?u11HRj-I#B%b@FQ*Brh>O#1739BQ_U`7JVm}fQlI5hU?_U^% z2H5t;tD` zr2V|H6K0U&^jO;PcEVM^nribl3%t$TR)aUP!JMISt_{_B*Yfig+AaO;oSL3ct;r*K z{Je#B;_(~aPX6KAwKi|F!CRl9WaNnOQMEOIUT_wyFo6*F6F4R0%dw{f`5 z+dS}g+sIe+@2RI&-M2MnE%(LG6g52>?dMICLE{PSFmy_q{4Q^rRrE+*(Nmg+;qAw# zT>2-QxB1|0{<;9(IAN&=dWzKShg#~P7~Z7ndBmk%yS!Spt{5WKY-yF>q;X6T_N zlbSr5Now*)9zAbT2FCO5mbF5)p0~(d7~Vee#b12Z=4}yp`{!*LdE?tn9Xbuk&zu^q z$wN>16quP`4b{w}(V_W6{p`WzjbDw)%seG+cpHD`zn*IIb|ZLu@b!$mh4$HO{%rSK z)d89Z!8eI%;sk)VRq{;8+Ehe+I*6{Ys z`+j+;&D$#Q*0?HoX1k z3upGM^0sl`eNOo0iGM$Sa?jO0W*u9)liJRxfg77umP&HH6|oXX7oy z1PpKA`OH5~cJ8+`s=RG~yasPlEizB7@k<`YnUk8H7|oo1-snuFrOugO@v`je`x?+`3E#)L{l83eEae3ocxr$ojvHa$KoATDp z_~-Sz{`~Z^DsKgm?-_VM|f4tDOh%d5OC9^Kym zp2qaXwVpmJcP9OLYybTg)2(K$#*=CSE^j$iH@qGE_G894yj@Y{ZRGlbcuO@s=F!Ye z^7wgUl_Is&ZA=d?Z!y_CYYlIU#(ij_!`qcr-a5V(z?)_o^zeHCYW6Jo>B&Rud1Hq$ zoy6pcS?lLb$}Gh_6EM8}@%Xh{z}u!b|9yQc{5*#(*LBG6LCw%~<#CN!>lsQVzaBVE zHp#MFLiE!`_7a#w^xBH^VS-?-F!zK-lY6eJk`YV&R+~~(lqkk z9qYQh@vEWQ)4Abo|6_+Pws~s<-X^r_nBV;tx)+V#v){_yrOA^YZ>c)e>_X%5^Omq% z!`q)uxnhXTTU+pU&G-P`6809vhGwdv9zAa@eb;{8Lg(i38{X!$eYJzl+g0H0rMm)n z^VCYhkMxU*JnH&+lRA;&sX7eJPrszI@y1UXGkbPwc>CkkQr=d$0~~nh9w7(c~%8W|(T;J!Ip}nsDaZwa7e$^0sR9k`LIt^#E_r&uLQLcp`bQ zw}-dn6EOGN8Al9lW%G75c)RPLHF)D$@2)!ev>SfcNuSC>@3-OSzWY|2w`;)Lf@`|D z?`cb9Oq#Z|m)(yy|4vC==&YF8v)0^i$80&^A)B|J;O*K4HF)E%Q>a;2(xHAS597>9 zO%Js+6UJTMG?`f?m1bA$ftt1GXI#=Gt*n`uTFRh|8{Uqdziq0`TQBg|V{;AOl=ChzL-bTo2Hh_8#8M%&`%w+R@!MyPM0^SinI%5b;H}HPwemfee&Mm?Uml$ZRUUa<-dH7}HoR@zuhRzme(M9??z=GoZ&J_vTu544n|C}v z-cohWiN)kBhPP6;hPPiF_&@l8HRXRV(-*w0d@4uYVy48knj1soVb12~jorrRVt6af z-|*Jsr@!c7-*5fE+l2l-;@{JlX@sr~*{~&D%imcJH98_3vrScd%SbElpdRp5)iWSvQ_HsY6x??TY?Xd0gJud1w{ST64e6 z+c4%zo3}yWZRJe?ym1<-H2Y7LQX4S3x`@q~|%v$CN)h=)BggP{Hs6X#o*?0@>s^mAk?f1b$onN000dHN0U88?b zLnoQauetLz9`@t%#;vS<9TB?NyqFi)!JfsD^0-MZ!fNS`o}hJ!@%3UFV^5qn%?ddcX^9B-)5dnp}gID z`~ru!;o$AY5k2+qX-hm2sb`nBW@aX)w?cV4waq7gZQpO#g16DP6vUf!?lF^$$?5XO zY}7FwHfk;VFNU{LMf0Bi;LA6+w|N@@-rDU5;H^>9V_i;?@lfsZ7Mh_^{?G{&!&_+@ zhPUGmcyATAy~=-HKN7rc8QshOo`&wO$Ww^vt>y9-I^ndud1h{Sn|<*&-ctpx%-bmN zwr+g`c=Ple^m})qlc(R$o7ABcXXa+pX?f%BtZBS?zfIrzjLq9<@HX|eI=nG|OVrj( zDm?pnYl%CRdM=c=zrOL%6q~m(;I04lz5VYgf5(#l{ime4Yp$i{TFFlj*HUvWwd7%3 z&l^{uF9YdZ%m*w2?==1 zdt%&ss#?YH#?JH5nrRfu+f(oR>oS|Sap3LoyA$xHnItDetyvr6k!tN8ym2!Aosv4P zN~q2Kwtd@wIPd8V;BEC)eH;6pN>fTWhsuk_oVy=y+#NMZE9;netGe}_uKc| zcUpPWFmNbzC*vpzNeZA=i$l5n`ZKPbQ_wJ zeY(8ytD%`ae#6_sa}IyhzTYN+w++)0^Tyw0qL%ukN79p7OI`O zGk7$g3wm;$eCC@!k4YBF+rJ!t$qJjd>ELa9_W}O*RGI+KPV%IROSPo9rtTizl25?g zZ$CI?{%>vGW`MVQ=hop(^PPsqBe5Z^Wt^HUF{k8VoVq-1MyOeZacZeLJ(6aAYE4f1 zCC$3-{l>3K8Kk&pt>NvQHhmVdKC^2EfGf4XyhLp9Tu zrs47?Ib*7navI(i&v|u?&D$*SHofP-Z11VmGvkr{S58pMBgNT+r1iXMs%Sh?Cwa$R z-cogIk*BX`jpm6e#W^r zRovx`RWzENXf$(b{E|oFO7dspjonhS3gabhc>CUK_c{OH$9(WM>**T2h3<$wgz7wM zX>yiNLE{YF5&e1RXDvx*<4wxui5uR=96f1<{hlrWZvzJmZtQ#N>A&WCn#%9qZ>cgj zTb@FB>$=yH`)uA8g10uy>+ojXnKc=ZJd8`HK#xXCc|zlQ-lVRiilKff592X8)AGhn z*kMVV`|Z%tZ|!UIwg|lKc(x92R?eEdwGDKV} z&j0?5^Z$w82;S}+JS5wD>fyX5lk`)wf5tVMYdI6HrHAqIDKH{A>7kbVj8o^aR+Hc5 zjkA(6NZrzJX~Wwc-wv6X+)eyh+Z;xZ!QuH`nj5d0PVBW^OBpx5&NJnTY&D%}jZT-3y##`u#NdD&ME$t_Z)JZdmjGOyy`~7#lZ1c7fyv^R3fVWhW zk@m|^F}38^#QnTUJCfp>iZN?rc1_a!yOwsLtZsN4{Ot6v*}Sa+Z^Oq9kAF{NCMoSP z?|6Cgp`FuDop-k}`LprH-BEiwF}$66;Oo~`dE2<}J|`T0;@^*-+;er0S;v;XhiY?s z0^B?;Wd1fS@0{5_=y`XQNB(TMW$47&J*ZQARKHb!sX~+2UUaP;~*a@{}0+L6QhaO2w`5D*q7E`5E z(cEvNkJx;W!`mrU-d1i1;7xkQ*(Y~OKQ%qlT6#2^d8nneE^pE*#;K*u^iWHFPDbOG z@<{P)yh+*Ep`{IPFD$vgwZq$~Ro>>d8R5U*q`HiAVvIL~mU?q}OVuqaQm4uvnn%xD zWVeR5_NVRrGl#d+s=N)En1Hv?3Gj2lwTw%eern0jIQ{wY#wyLAUEVko>d0;lZzCQY zbfClA=~dpk-&=z>O%JI&+@tjQl4?nD=9F|iZ&J66$8^ZfWAeDXu|sNBl(eT4!`n*- z@4wUG?TjjKue2Mfe@{bulX}i09@<+xZ+Yw@RZdRB$P%+!|O@OIEU=I!I~ zc21SIXE)X1jX9-Xn@Dl44b|3vby#!StgG>}hw|jPieF`BYI>;Yp{6HPGY>WMFfQqA zyh+(OG3ruRhPT`Q(8K-ntaGcpZRBtP-ZP#1?Yt^)_slMcH)YR-v8U&apLAjrL#k2skul(&OEH})+KZ|7He zTk>c|-ngUCiScx(L-R;}O-?;;Qk5d(+!3{=heCP#+UJk`xx?E9Ro*6CJto_G%3V`y zCJ^dje(KOX^y_&`Rgrn5Pb)p_Hb!UTP3lC7mpU}>>HmB5bLTm{U0CI9$ox9IdGc4@ zzEBVS)S*+=tfhxq&l|g922EV@Fi)svPDwMTpEo)+*(84{Zg~5}PgeDFc)O^|+rTI4 z@Fw-p(&O29lkTNvo`#<{&5ksl(gX}|kDv4geq7D2-|xM+%G>L`#%6m@n|Z>T-m>wg z$=qxnKW_=Uee-_X`}aR|?zc;-ygjyP4{y8vK6%XKl{Y};JCK-Kp=-18mSIPRw?i*^ z`F9R)msWXO^JE>~_+C?o?pou|J3o6!d%v-BYF2EN&c<7YZVhjnM*QJahqueByv^=+ zo%^0P+r%>Lz2;5AeU(pv5o%3E$(c)N1(jrixz zyS{(Byvo~zr8RitEanA5+V{-!$1Yp3q&;pGST@Z|o$4HoVPz;ax8|_uCa! z-g<1!$Xm>0_!*J*CFPgaN^vQ_o;OWq$s@%zyVI$|t)rwgw<#c(IoGm@>=PlK2OL+`$Cw^xu|33MqH~)Qo zEBri%hgK8>T(ce99^jHszySCBdmg={dcl=>YYpC3K3|77=>s6e^WaMJe^as_jX(Lc zD_1rIEy%+PV!9#m->eBi0A-YTaP>IwBrb@Ss* z>QIWaL&+n>HEXl+CROyr4R8N=?F{^bnezSC7Q79-IU{e}X^fV*)#MCaOMia6v5ICF zd3&<)mZ7UR-_w&mcc^{8T?O7=-`-%}q|b}>mq#o2A?Kj+(9c?a-sq%eHjUP-m3F~6 zHFIioJa1AJYXXM1(`Nl!cbm6%;BCjXH)MNHty(pCphwelsE7F_E#>s{CS_xsnHdkg z-$MPYrP0}VlXfS?OPv_rW`E#k9c3QQSW}}uo zQe4w5^T%jCZ|s&@S>5oq?YRZ*ZQeS7w+*k<;7ytad!SbKQT<2*@$<$E zn(C5Aic4MP5zoe()CqUQDV4P0ZSe2E{8O8^j^J(a=<)jZl(nROT1vcp3Y=K8buRT5 zdJ2Btq$-S4OH(R&3~&2ib9)b)w@%<~!ukN-q*{!#H;v|6YOc*A?!Vtyk(ymm)02nR z)S`zv8*ibzC{;APU3Sd258J$T25+NYOUzqnpOQcHQ!M3?;@NnUvN4`V=P^0`ys>kQ z_H<%+>pbXylWpF*fVYmw&luCI)A><6})Y|qXuuh zjivmd9*tkB#kiD*acXI8`4luxW{@=VP)mMIoO!6ZR*KV;mN!=9T~~@5-tKz*Ec}4W z@}Dns18l%8o3|d|ZNc3Eym@NX{k^rAwLICF zJY4I3y7Q>QijC5R^0xN{JKNg4T@BtwUNuSop7K+}>1pDUN6I7l8K;)=NOAxD##MP} zsq;K)#q%b0%Xmq@`Tg6iM@_PMy9T@snjFAe9@A)4PWOImR3~}Yjpt3-q2X=JSAPGb z&0A0K*5>{iyjk2Pc`IXHE6v)_o{Qlvv@7MwR!)TZrL~3f_Qrd5{MF{I7kGQ7!({z? z${k5NWt>{W4D(BxIVF!K?()XW)S7G>KPMphr8w70{ctV)dfu2pQ^oR|`|Y&iR^!P~4Z zQ{4BIY)hJTH6Hq@xmJox9{MHCwLCRFZ_J?4QYV@?^GkkuB<=2$)M+wH9w}bx#PD|V z|2}(!&09b4HhESJ-lUwnQ(Vs*cg=HTJXEL3qp3^Mvh7+^rQ|WZ?f2Y&?{D+gAH4N> zWDjo%en67)L@M8rhWcapER?s8UHIVzHg5yK+s^J&-S;$f8mas-Qw{AQ*H3z8uB2VW zOr}uY&i&md@B?ege_lTjygf9xiFlKyokzT7@fI_ILV5ehz6)=*@3%qVZPjCYcuV}A z#@t0QTuEySV6NVS*!8n@o8etw7fC1r44WM+Adsg^EMQ`?buupZ_?zYxV4wc=}Ugb zsii#b{iazZWn-LqsOgE(p*aiX?Syy!>H?d$Vc_kaKGXH@sr0*xQap63%oDm+@-rv3 z%bS#eab}3o%$c|L^Tu6Bok$)lZtk~Be)7dlHgChh+xo=;yh;7E#CSH|Vs;vOFU8d2 zT9>z+Ix)N*bIGWKY~HQ~Z!@2+!5g`y<}UaXRH#G!%&F0w6>CYFy-~Zo@vGGAN~%H+ zwZ_j{lAj)p=9FCCqzvp_(p<}KOMb)KX>B`w$mVSXcw04KhWT{Pn`T=_764MuN8)%QN!ksayHpUfLH=O_N9R zGcMJV;x2D7*`z8`oVBPWe@xuZn^awMF6^q5$MAO2l=%*CqrhAL=W6iATGY%>%{!KH z&06{;%{)>qDbBUbUp@t{qLZ2)YG%`DDQDXPJuYwTgidN^qo&8whPR)8pseA9yFMaOliP7%;7L%=+*Sfsr)UDy|$LGD}7dCHWz+3xOP0E`z z6AgzNPcgh{I+1poDqak4rJWkyuKDo)eAVV{EO^_zEdg)R{mi(eL#LW5PcghnyVG!( zcBiCH>MAmBc>Dd=A9|b3+jZdWuHm!tdrvh}jof41H(KhUvAm_)wX{3NNt%Aw%%JSn z@OIo8AN;k=+x6gW&YB!~Q}$VxnLP67Cm&i}b1!L5lX;{bG#))~nru=hrMThkwA=fi zY4bJ?yp4W20dJvwO4F9&%rAMQxHKO>Z>%U~U|jM`ajuQgnq9lRu`BA(POR#Nw+GHY z;rljkH-NWpqh{y#o@)BHdM@m)^Yf-&i6tqX z(;R9tyoWdc1TvhF;qBe${L=aRB4UPMG3(e;7 z8{USDo%0Wyw@Kh_;O%vI(@crqbJEYar_TyAp&r%>T^r9^XvLUrSt~S;p107%r@O^4Qm>)!C8T}eG?>e9oyE^n+tC$;2Z+|q`( z^A4Nr{CU<4@Yd_zjJ#R3>Q27d{CeJaj?K0UO+S9#G#Qj0!`r+UZ+*bN-)4fh9qs13 z@2QlZ{8Q7aX%>{2$X9VzO@m%3N zvd>DMB2UNVE!8fXEl;7mjcottoi=atz}tX_0(fh-NyhZkJiO)oDf9D|u)4Y5o__3m zgKXaBgSR)jE=>ELN_SwYcqn61-SR0&UzOsa)${OcdZWkBo75pQC_RR^{%`-5!`lMz z_WbOOys;livrl>$mvpFK$|J@7_ZzEIGaKX7^n_~Wku-DC!#K6e8&^?tRj4lAlZLmW z+P;3VeZMUPZ%;g0hqr|HZRPv4(7n)~svkdZp%qhg#k#4q%bO;1h8-E+e);PE`+&{c zBJj5Q>P2baQ_WstJQ?OtlXE?9G2J%9T9>y}GqiSRczfp6E7#b(-3Z<$&riTxPW{*9 z#M)zBkJMXZd6T*d-JQno@}^mp!DDzk;GV9}+Pp0WZ@r(W!CUAwB)=5Tu$Q`Ah30g5 zE3#WnEsaOoHP@!)jhU_LhPSh4{p}+*Z%e@23%zd4_MXOkpDFbmlhfLJg-_m$c#S z=xbO0o6XyD@HV60V)s3@^3-I29?dC)dSdc(t@IgjdE+YTn9ij^laE8?E zwiGwKJvL>>aW-!&z}u*$HF%S9GJmMnq>r?x0vdAua)vkaqimB8=aBb@b;U% z+dpOVb`yAOyR`;yk##HgAwN6R%+I)_>DOpYp7JSZve6@DW}N;|%{QCGLC5{F+aY#>1S=ptJGT44sGe;PS={3D3^(_VmS9 zooDm53cNkCq7H94^;Wq9nZHO~Ja0T%>O6MM{C?gv)tSxXH@qEq{zCjb)s6e^bHWiP z{{8sLJy-Xbb!_R(mp>nXo2OK+_$ap<)I(!&lO~zpP6?ecg)hPUr-zx#2Ax09>9b-THR@fJCg z3jhAeOXr@dx|BaOuIb0mnXY1#U z*{G$=jI#>kF*+uvpSPH4WB*!=MHbDR(ZSk+GYOtl+%_p>q?W6;#^y#_VcFc zBy>lb>1pcHqv^rVnZ>LpxTe`LmZ&H57rAabQe~eC*vwRAi zHZxOeJoHm*cB=799!=cO8#73nyP(I48{Q87;evA<-cGObHvg46ym8vhAEQHahWa(V z#ouq5DjH87yJKBHZ*)d#!`rbxzx!tnZ)a3_8#a2m|2@_8z==_7);7x1JiKuNoE|kj zp}J7szIDdI$2+{8S>>(G`UJd%P9T*(=0u9;EvDPhiLq9Gys=8DLvz1%`N^NZ@9_4I zDsTUMwFUDQdJ6Ox&l{_2v~`-5UzKat(vu%=tWwg3w-0ar+dPN2v#PwUzHUYQdur9K zOh)?Nl74z5E%`NZdZ_)pu@kP6;`C5!{2C8)QcG(YcX{JinT?trNz-4_hPOE{@ApxM zx3jCfExjX0-mHEq95NpBDVEksan|+oMrTXcE^nG0Ni&q0oUX{15cL(s6p@*7pXiOeyKAOq1u6w^RvnCt;%*;5ow3mN zKkM*zVU@S#_toGn!(Cc)Vw?tZQipozmo)QJ(_cOX>8o^7Oa9P!XwI1Ye%_?aQimSD z;q43aZpVKwv+F&*sLI=-4y)YvlsT!T9vG*V)=D0YpKC+)75&ezd@jnTz=)(%b=9am zE^pFK8Ru*@otHd@x4C2I4RG$ai>tidIIRY6tku#r-v#RODKx|G*cJPc;`B(GYhB)? zRhl?6TYkgaXMcUV`~BM`Ro+Hy+|7BjW@6oM(pMSxOt_Mb{$|r@dGqYf@b-=U7mjo8 zw@a(Mb?kDp`<{AoCYdl#kUCXe>HcRe|NX`c)KZ7^P_rV}(!+Qj+RvNRAv;le3~wL) z;;Q}*Z`^PU-G!TX);J1axE*C{D!xm-ZAMU@V4pAe_!7UKhI&rV|9RI z0cxos3;N10Fr;{B9?8$%sOe|i1&&UQW;TtVo|v_vlhgyp4yjqu)42g|%OPbWI|a=wHMZvaksI9IHZ})ug)I|$eT0) zPtO%@xt3Zw9eS8witBk}25MH3wB%=;TFS$CsJ|HAN?jS=2F*I7wSB*J0B<|{-{QWf z++B>8*oeuQYOS8PR23VQrx@N!yED9f`Jq9N*t~TFZx1i4!5cq8lIHXz599Px(?d;9 zD$V@mQ(%Nzsz{Hdxi;n#C#}`grN`xsUuCzFrk~pK8{W?T&4D-CymbO^_di>MH!Dx& zjg#WiWF)^7PnC!HrJQ^2x8xHrynXoH7anc%))~CrIC!1@J(bRm=S!W3KQyP*Px5Kk z{Zz7(JhU_!_kLq_X%#(6ZFoC+?InwB-nxLd;VT1p)6~`UuklE=V%9QeV|kNy5gO-? zsF}yl8&^eY!`lys%(~9ztt)tIwJkAkQqO$%sU?3Nan{xI##MQAm3PiUdHdAw-|PH- zNH_5I)UaFK_f#`sjVET3{Dj8jk=B+^fmNubRiSZCNy<;p1^6dDj7w{!e&~03V`gUH zD#ocj9&^7PG`jyZ`+n;V-fmr8gE!{iUA6vx^Gvwm-?M2>$K_44Duc)HcH*_K|Ch~M z5AZhgr8>NE+SF1%8jrNL$ap+&sdkzwk2JkPdE4*Pvp#9_b~SkGH*&rEo=Tld@w_J{ z)s2bkd5fvi3~LMJZRC4~cd&W82E4s?OC8?$)>DU0QnOa$Vci(c{k%z?NO5)*qow@P zP9;Bk@be~RW(G}_lHa_i-}uqm&)K~71aDh*)ZvYFTcVcc>F2GPcPi~j>ezLc(-&6J$I*n9oL-Q2FTg)`LqZnNbZ>60Y-liQn^BSAC-r#N8 zZHak{nM~emdHVVB#wygQb{Cqn7~V>|GrS#s$nTtgzt9J~^?iLe;Vs)Kq++0%^Q7g? zGiSrwcdr}Y$G+eCg14R5-|oJr%I^%Fl9U9)=(*E6lvU%$V-nQOZgE#Ij^hCHe51p!?@+qY1FjZHYel)e{ad~4E zIvLNP4R3q@Y4VS3-ui>L^{wx4?>E-67_RxLr@y6WJ#U-A5rw?=iCs%|#kGE_Ib&ARjEQ*7P_ zg16E41n@?lB^}ylOdhV4>e549J_SZXGjMIFmTF0H=9fHD+~rNW2Uvw#%1lp58{S4; z({Z!S+aU0EZM!@D@2Rp@&0F2-r=}++zqD40v!8h0q$-TFq7joW9`?q#q(l8u9x0xUH>oSesio>MaptF%a^}aIRnhSF<1;^dx_!S5 z1#kCs+@OC?rQJz!tDm6vH1D1A*Eyv^G% z@HS_90B_RUZPiV}l~jw9r-zyzN$Yu&GD~r(qU4vRC;6rPQe4j)JIPSp@U~>s&Q>;W z!@=9=O#!?~JBo>Ok1=^d*OpHqbd}^Uau-r9DX!;D+MP0PczgfmPwcdLyB554>Ux*{ zJ(cQmr__?a8RBVqD{@Dn6V~$2r`BFF=alwR z3~y5PnB9fu)bke8RcWo^ZO*i(w%fdo1aIpesli)h-OOxg>V{66Ik~nN-dMd+Iva1C zm9lfg+tuH@zl+V=DDXDD$K7e)Q)O=zCNu1-vNp69{l)OssNK0cC3XC2LTz|kc;H!I zv3VN}-ull=z+1weD=dYcl;oG<#qh?-NS%aw=+|hHmg}w!Z+Cp_fP-w_#(=jSk0s_! zGgbatn_4qH$s^U3@~53Vt4N(lo#!2Qd1JTKQYX@mxVGdmye+!;cQ4qyjRkLy^t>nA zdn)xH^_(i6s%|#kQdLitN17`0ySz!OBIAa)sSA2JKfm`n@V0zmM&6{}xKnD$->7(F zd6RaXYGTY&C~wDqdc*Vf{dPTgyKYNj-lRzu8RxX6y4iS>vgIAu>`3aNP~KL2f9*$X z-o}BqZhh}{-_w}gNxNn|@BFMK>BjQL2~bPjhQ^uG&l{bgRV=@GPuqN}kMsK>H-NWi z7T4g-%2W9@W2!j6&y=)OD|9~0&z$8`kiJT1s1D63`9pWgKDpM<8&^r1+32x4H@tmv ze0%5D3**7tJx?d#P3nieQA>U)&b8E`ey)wb->hjT|5Qr5lj7VRwVt=o%&bzWRLx24J7TRqde(p%hpN%&uvlK5?H@w~O+jn=jd7A{@+CLY- zo3tY--fW+A_L=s6V`fQn8q|_MRou^8s!mFI3~%2Y+3&w?-X?>$ZG-Q3-_uk(4b9{E z>{RYXjYq1>wY$gtChe4QKW}tWb5_*!SlaM5{B!-c*t|^vZyQ!-b8`)pSPB>3r(LcZ<s zdjonHr=~~J^i$K%I5j5JUQz!&{Ed=dE>OHHMWn_L_P5 zOEzz_!Q1US6Yv%@;n20AerqO`&x@ZoX;qQ&v^ynrIqkymw&jSU?zVZG1Kwth-RQoj z>_@t#H6G1c=D9-n-=j2nG#=J!e%{!vW_R>ZleDttS6L-e8{WEh?E8kz+g$KA{`QQ# zv1jVYo-6l{Ca2_);_R(FdCeU{54ENWE7DI*Pb$s)E^nb#n2l>WjgmIJJ-qRX{cPUm zfwuu~)ZvZ!^U&-$kNi^3@+nAP-JRm`yeZGf@b<)~`a6HVFdw|VF>X^a@2T_|;mLAe z)bub9H9Z<#K82VJF>Bc&^H9?hs+lJ(Z#)-PDRpRg`|m>!>1N+=3&7hm8|v^Dx>Jpx zJyVCS)%Yb(^WSfwlkxAA)Je|BxZ&-FJ~#ZE&D%ooc3Yc=-1{xnPBr}(S&RMnd23Wx z?2tWB(<5p6Nt%DxQZ~k!nemb~ynTA>+y83wwg|jUomhi6D^KNn_&nmglc{5Bd1g}C zi+}RWKrMAB#WlN#S*yvPmN!pD!`pw}bNk0^-fjeMgYQku8&Aj6TjksvZIkj^8=o=4~-}+u816_dR9K-BtVVx0qd9rx5hXidoB^ zna9r?SA}X;DXlfU-G9gM@7TO80dE_p1n|b2pE|V9BK_=>IxTP3Cj`J#R5pN}U_tZeG=4lg-;I@V4O5#JolJTzQu@c{CpB1oga08JaoH$>@0t-4Uyl z)*9Y^_R@{`_s$#l-RFc~o%r|TC-+?4W7e^yvmJW%Ki%h)Wv!GI9O%*LRCzSDG#))} z?3|ia7^kL3(vqKXNlQ5y&kr|NDd{(Hn|1*HySBZ`yq#3#ZRfl?yjeLbym11OCy%(M zuKt_oR2^z6YHBf0+MSX*O(z*VhPO>W9{ADfbGnPSldHTv_IO6#GW1hnHSe{WlhWUB zd3USHui255hot4YYs1@`dlxTrcsr%a+nQdF`QKA%7cD)W>*NdjWNA*JP~J`&dG>h@ zZ>Lsyo4%-p@fP|?r=NQFJZX9JbY*zE z{o^a|cX&Ia%G@=Qf>bdkbE>tedP~cjrz^wT*Sz@v>r%6SYI)^q6Q`1h1`HJ@U7IGG}~d%rb9S9wgpY}8tpx6n>B6VP}<>xTMW-a<1d{pNmq=U?h>+r_U0yTS(rb@rY!<^L2!?;FsEw#%Vze+7tp(kc{(ptt@i~UG(dR*Q@GqVcU zvPwxC-ukxt?lgzD^Q*kAxpqtZdur9K{CcyQ<1TN_)~)AcD-+guT;8}-O$Pcs+VJ-G ziF2=Zc)Os=+my8dypeO(Wt^HGNr(ESJd86ZH9h51;I!$C(X7Hej8lht=$AC}`+3u3 zkUUo0@OI*o-uQ8YyMBFoVU@SZuhikq%31kjaYxOd*{7zTV(vGoE5@lcol73(q4x78 zWy=sZyq$5x%qyMy?V>7g!$&{qzNh@uXQOmZ(SZ=t&m^|Ly4OdieJLV26JXv)tV z-Y%~4)@FST-Z%|O^Aki(KjYN&P-~_~k0z(&DW8J$Rdz*9KXs^weo0IDL;Wsq%q(eE z(RfOJbH9!J)B~#=-Y%)~_R6and1F6~(qu2R2lsvp%@FFBx|QP0ukp}N?efO2YP6<$ z$zynX?YP&cJG@<5wi!A>k(;^8jmzRuBAtc)1%Q+o_OA*%bSnIC!XOEzzv zz+30;&-vd|D`(wblQx@QI@w})lcphcCHZ6Gh4Qwv`}=-i^VS)>ZJpC#-ZYcfc%&1N z*1EjWNiAj8#AEWrtYvS7@;3NSzxt-lTNm)Q?w@sd3*EK#rf0qnr94uc6P9}BT9-Fw zpp)?;wbX+YmwMCl#>~`GW-D&q)2BM@zz?`A|9Mtd@V4-p=iT>IIxj1KW|o>EkDs?@ z=up#x=A`tzNfk9!N^1>oM|Sw$z4raq4ZMw5P=hzAt|lJ(xuBmqw3f7%an4Q88#7Q# z9nwRc$}i=S;x2EYT`{v~t>NvZ?K_?C-@1dhew!2XrtGb9H%k38PEMGgTH~Q#&l^{< zE5@lc9{NK&m)5c_waXhbNSYZWj}(-dLSZ##u$uTwBtHw;zo@1pnbj`TMu4 z!CU*qHF%5p#Ic8%JfUkt{n_p}?yjY1m$#fcH@uzk#&-_1@3(8f+vcZg@D|z+KNpf_ z&QLbEmYTH~*XVfOVyd&lRC$<_+Rq!SQ!|578{Xd0qw7gFZ#}`=UH!MY@2N6>&G$4F zOEEoY*2eRusX|YqCgbOg9Y$)y+lecWzu4xj7kFE+ECFwgnq|^xR8+!`sw>i~eBq))%~Oxv36s>?gDbu8q;D>N3wB z-jYwi@b=dSkGag|tsi*1<%Iy=Vwh=$wav*JyP{4tv26EShU$j5we$bF+2*Z3c$+rt zMgM!6p>E}!O*K6!j}+JR?0&jqvS}(d%2OzBxBdJ}f3uQ`7oX&`ue^~H?5R4cEPw+ zxAOfPJax;xa*QO_uGRDpFPXwZ8&&);I+iOX-+4`!|z?G znUis9dR*T4RY^0OMoW1Zw|=Ts%@;(f#XQvX`+1|2TFT~$8{R&?^b7mgyj=_4ZoB@K zwC^dIXK&P@`{LSA&05sicw=>Dqo#*ie z?###=YjM|7|B^o@F0Exhl6HAxHc2y^#uMsiPHB=-PR6;`<&9ruMP^`Jqf2WIZy(xt z;MZ*4MuN8mt#`Qh8~f4ppz*M0$uGsZR??ws{k(-{ll+WpGC>ZZ&DA;FKMooJW`x%HJW+oXPlaz@+t7E)O5B)&Aqt1@v8}U zXLx((-}Zjk=4~{18-Gs#Z|p6mXX%ru$*-v^c^G%^H%)cP!_1P0yO#W1D`~FPXgzPt zCRO*u4Q~&$yYMxew=v*tM7y2td+Nzq`PCDrr}1k%dE}Jp`g!B3BD-SURNCb&Rc0%X z;cdt8*GAjCjRkKVrqtmrlzEMx9%`%q3fr1EsgQCq?()X3YAQC$!#=6m52xYsCaq!y zYGz~H(uTJIv+r~M{lazN?Zx}+@TRO4#D>*pL)N;yai`RpZbSW=oEnd&uFIPya|Vy$ z?bgGtTxj2K*MqmaJHG0^rYIynt4)b=68AHS2YziyDNDNZ(se?gZK~I%0EA3 zJa`-4^)>fBWe=KK^tTjUK84+NcRU4`x14ruc>DNq`;C)nHtq73Q-_AP;cvY04f{Qv2;R0ll99KZdalV^9<}_uk#OofI>|d{p}c+c zf-{|eUq1=F-P+@I|9i@5G~47gf0Kv(QzpPTQ~fvE;esdz+0ck3*s%0smA1V z@3+Wl*ZfX^9a85}OUhX&Z|6R_*La(^so?FUo^R;iQ)x#nF<$)rmdBJd49D}PsZ!dV zx!(r=d-wfq-ll=K2Nv$;yh*dt^Co4;gC9-K(0<~1({#0qx0k-Y)%o?pbnv!hO8{^D zeM0Kc326Mx&;2s4(Ok>Adfu3UTB=S@sAkU4wUWP3-j=?juk+`pGr-%FzO6oh|AjpM z6r^3}5hp*P`TaZ9WY&13t~6^k9w|TLh4S|DC*Jx;`#qfr-g+-d%$xKHWt^Im3DwL) zt;th9h0s-+%td-MOy@CNXbCPCKSn`4p^))%-PPOqD#= zvS&%NZhpM6N=X~uR=ls5^YbldgSSTpv~JG#l#{gfm4u4&Ddds4*>V=j+r)9#47cyM zIpA&8@;bapFAn3>c}zPdKi86BKW|*6*%32q{CRlTGqs*Ku99}dICW`9=6+k*Z%KQb zx4GbL`g3)7%VRRlkh6RW*2FU3oII&KcE>(hi<)&A*Ym~<)Xc`Xr44Ug-gUL}{~65# zZ^MSPX`c6#Q?>R|AwlDj)-vwD-{=h0sX7d;EBWJjlQLUz!`t3JJ;V9)h56vE?W!i? z&Fa6xSRQegH|3`|b5|0FQeDX(I*n|+NnJ_ROMb)K`hWlQbM||>0K7fDZ4Ym!y6j0) z)x^mOwX{~_k=APB*?7}rwmffs|Mu=zoL`?V1aIqyw{4#H)apO;O`^$@mN8b$n;+KF z=t6n>x0T%;-WGwk*=rK;#=DnVGYxvUmYN=F$s@(NHXCouOwG*Ard{6h=)9EE+;2~= zS^h5jJ-rdUje9vGZ|s>`Gi}MwIJG9H#>1N@8*i+_4w;#8`l;!mrbp7DewQ~Xb7*`Ke@F zqg~!&vPm75Ix)OWI-;lZ?@yP4x7T;p;4P*f%>%UaZ_{LE861*uy) z*@>j-r>38AYI-y}bgj#ql!0+-cEz}*4Q~e?e0wXKw^iV+OWO|d_Z$0ZDY|?L>{eor z9%@dTaq3ha<_y((-k3Rb8l|;{x6k$XuZOCxw^-!V@tnJwtZp( z+%mM5>u=F%`f+jNuy-fz;+E6y3~o=p{w#jEuveM4ld8PkcW({eH0PD7=a^cW%R}nZ z&s$7aQbl%4E%{T$UEcE8U8!5c+xsUDKCgPc?c(j^DsM~Lcig?7Q=TlfpEu1}GaGf@ zYnh*Qsp)Zf<5x9W;?t^Vc>AB<9<$xy?UX8SQ>WJ8P0BCz7V4MsXyP%oxHg_QO}C*Q zb{M)=^2hTgWwyFCyj?SL^>l}~Q>(o7dawp>QhrU`>Y?&|niSXM%)?VYg}l4UBY$X4 zKX0Mgcox=L!`tlfGk)Okc3PFUZJjz5|DIZXCTCL0&spVnziFzole~4IytSUPrKiK& z=~dqDoRN_?>+P;E6B_5VHEZdSw3L(n@+mMvEoG46(pt&Sc&HAo<>yVxmLYELx36xQ ze~81|8CBkHe<&kwp;Kx$e}4B{bMBPXX*$esOB>$a`1WD@I=r1(A(FehzQ{sPZ;&PC>j`y;a`dM#c4$ zZ&Zh=>Sp6DLv_R3d%t^nXNR}5s=T%NXJXzo?|z2QPK8fRevPMm3X)SZZD}pz)Xd2V z&?9NCl{DAVqvuV^z-;tTmpq2I@AsZv7mZU(>%O!r+H|X zH>rwd&Yrb~w|8$?_FoQf=T>=}`$Qeyr2H*CUJP%V6V!NOrtR_;Q_-{5@b*Z%wYNCD zomb^;Nbj!t_tbi?Gv9Sm&&;p!&@Y_`^Uz-mZ&Fu`vn$4_rCrb?X_q(2DaEA@*;Q$+ z;ceMTmptb1c7Bz&jyDGI#(u0?nb~8`Jhaqj^YE6(u9-8QH)iu_!`s>${{8z7Zx>W~ zd*;cEym{(Y*ptqhYw4k;hg##IU()fsrRqfLFjYJoZ_@6hc&S^%+x2%n^=*f@3#+`{ z(yyETJxw)%Jo0$vP~o3-%cl^!iWySrynD#TTZY{k-j?pQPhW?(i>ka$TUror()R!v z_EnP~=GSPKH?B(6RipB7FU9ays%Uuo&bL=>ad^A9%G-#o1@YFX$xC(1ryzZGcZ#Rw z&2t+Y-sau8^!E;LmsEM{IH{o{ny14!H9eu)v&Wiu9rH^)6vG>8GYg(k^eI8Klfs-0-&UQ%B*)sqXsotjnsrJ@ zKf8@tE9G%{qf?`mT^Zi~eP+jVocrzaDsStD_Q>|0N`3OPLoN9=aZMhLhqcP5pqXT7 zW>%4OOis;O?kgK_p;bydHM|}1{ioc24|qkDw?#Jx@Fq=9igVYJr%`eJ{Z?d$p%c^m zhw3hGnkpKPH5tR(>Px%f$G`1*Pp_=Xg_e83`&my?%~T9jjIAz z=B+h&dv$Fc-dIbT0ORyi(<5p6r4yuwTJp0N{q6b6320 zs(rt;1#fqczQ%n|$+Og(#v`rO#F?L3vzDhXtu3E|^i^hHMJY~?lvCr;tYscaGmoD) zX-CW;RpHu_-|*Jsk55_u-^Y-&6KOov^o>zk$M9sWf}0c6ke}$ZUD6 z)#Rs#n*F%ENlvb!hngNs8{S4g{cG#z_a1p-2k_SR&H&!5{0(6yv*8a{z+yk+=w7~bAG=UR z%~Rd*Hge~n_uKbdSMb(navk27U((XuE5)TeMaIjgP-I2cikXaNEj`(IW4B6ecw6_x zu!C&gx`DTs?yJF@hHYgJb?-w?CRI);5BKH2-{{n2ll+WJJEezOQ&;jZ?&pmashL62 zmf!I9t+%ea%;v2-c-zvUkN-VoEoov>oNK8={i)V6Px1GgbvkuFiR?B;`+1AWY^^oC z?fmGn*KFQ;fVW$w1@LC&FKl8lbtSgb@+MUY?VNe2nMaD#LtQ9ufBC`9_yL#YzmIe^ zc$>U2F>jm(H7C#vy0N@zCKKb~+HAZr1J6r}mptZvJ8jT|AF%JYYrtFgE`9ayDeE?~ z_ROSmCuieLleyU@la@En?hJ4LJLZQAZQgopd1FOtPMdLR z$&;#c=GSOHZ|s~}%FJERU($xR-~N5_mn+QTeWK5ZyJ9p zkES1bT;BLqYNC7*!d?VN$z`rGg6K=Afh zp8@{&R64uRIPYG~+EBkHCq2ohU7=d)oV2r(ymkEjChf@T-0*hlltVvn^EL>)tzTRa zZ&q)W)5$w7opOG><-McO{PDagD;nO0{rT7oiHwC zqsP*Qx0By-0e+xr`R^kQ18- zysaBN$p4;lM={fjS<4<`GI9JsKUlR^!jcnkA5o41kRt^e@B z`u9}YbxVxOXJI1N$Nu^We&&FGZT^QbueDB+a*u0GbZ?CRS%v-7{ald)w^z)W? zb>`1o7s}hk-}~$hHgBWB+oqQSc+2D5^PaYpU(cJAA&+>f2{TWjynXQFJJ;E~jR9}V zMh;2)p7O1y4n3WiwVGNkZ>&hI$;R`Q*3zTN8S0Vh`gvnl)S=Z&e)FDwt=r$XPIBI7P^>{g@Mt>iCx3~#*$PW`*h+c@y{ z#%%$-v96?f?vf|(xYPsVE^m2v)u{X~Z#i{hc-yd~9sUFR@}HmH0Nx&XeK+JyGtWJ| zl_$W@pk{6!kKyeHeXs0h-*4l=+sbjn-1pRTXV(3m%F|D-YeVOz=Z(`ywY!*{(po95 zzuz=7WED#r-r62{@)b636TsW74K;Yv)J-+{M&;4-)~F6s)%Ek1@T?7Q>z?^`51Y4% z;B82o;qLvGYNySd=l?Qq2^r&?Wq2ES&A&K*pL`N{yJ}*CdCO_+j8I27!q9GlvB;j1G}ZpyKX#h%5DvBTQ5Gv`FS-{!Q0d+ zEtogyq%`v>o;Ru6(70xI%&F&%nW>r0qYZCItvhqQeZNfuZ+#!g$XkYfGT#iIJ!a0C zjW_O^I*$%h<=68TxeLSF!o}}&etz$C@V2Alh~{}uQ%zViw`{ygnTzaP>OqPZ%G+i8 zZ*_it?+oy^Y5H!+n{@h&>v`iUo~ab4rx|oM-ZJdY+;2-xyV&`8H8a86;!SmUB+a$d^fO*Q1xD!9=+L$FYczYHUy4f}J#SJ5R`h7Y+Z)fXasK`3T=2HD$0+wb z4XsN*wbUE`cC_SYT++;=sYQSJ6!^*ISE;4yQpM0X^K)%zTw3e$CS_&@YWg`bdP>^x z_S0)SJO6%R9(a3oUJc$nIg{LPR-X;Y>GIYLU4{0bIVqPn&8pB&N`AxJBagoLRr@`i z58gIDUW2y`wHm@qOrPabh{@1wYbAm10eD;9Yqa~GT6vn5 zCEoMwSrhm3#thU_6?&-YZ8-fHx)QP0J_e=I4#h zmag;T&6xY zJW@{fK<)CzuQFSanmtHb(+}6`dE+Xn3O&>%zq#KITlY{0o3|z4?XhP9c;g*QE%nx@ zxHX5$eaL>wry%V@lR@$@r{rNL8js|c*0KjbZ&C(URBFT9O<#HFoi=Yv!Q1+QWBu{nAuj-Xv#a-0-&I%=drA=50B68~%I^-XiPP zF)K zG*6+t4Z8a42iUyb1m0d*6~G&JDorWxID4RGUB;>D(euVtEm0TBTc@M8IsZOuC3t&m zdjM~FPg`Pxb?Kp|N29ax7E_U3#pDTHn;&mhC+7RN{~h+j!|eNQ6?nVz+Uv8ur<`6( z)Y*7z)J`?uAxORD#~Uk_IyAhs|LQyN_mMX4yUz(no%r|TC-+?4W7e^y@9XESt$~|` zZBu>wuj!2*NtX|SU*+)f&=N9?Q?rNAKAFdl8#8M(yRx*w?L&h$TMe)6w&m_b&3zUnp-|e$(SDhqqIzygh$wj=UxO>{RYp<_tY4`t`iAA~mxy-fTK8 zZ=PM7`|Vw$T3z7qc50Qk4X^$m;*BI^rF?R^@HZbvNYqo?89XJ&_{)*?23m!&J2j4!Uw$&>fm{CJC*fV5VMb4rp&&zqD%Q>ElFyj}N$D?2;)+drzjJ$P4Q-q>5H*6f!# zrMsOTDQEcc5~Ya{`XYsfpO_f z>Cxy^c{H`kr%+@?O;>q%;(1ecWq8}RX~i0cx3jCfO_)@JH)Z~yUvK7dvN1W!r@)=^ zPGDTpl3$8T-AaC`t|rc$E^quQvoX$Y8K>sjk~X}3XyZ3WJG`A!I_$ z+$wLKJ4|%nQ%&8No;CUN_PF<3-W5ahr<$ zVb|}6oL}Ybr3Vx6mgj2qrQb;1ky>z2Fjw+pMhEu7h4-tssR&3yd4 z<bF_M)jJt&NG7C(mxFnVE4( zYy2@DuBB!_j8nV3NvlHRtWsKQc>DkCop-pE#kK#9F`AfWZlXzU;*H5oVtO>Wnrf~w zF-2ppdQ(il`6+f|FW70)dj|!j7m+F`(mNuGqN1Ww>|j9!1p)o-$>urx^UUn^?ltes zyk{T6f8S@F@2s_E&1cV?gLrt)_Xq!dj6ywS-F)U2MN z4R70CAMj!4e!H&1Tc6&O^zUg#t*U?KAo;mR$>SUM&6$0_`BtxIe>QIsJ2$+onE&Dw z@V54!|9}1U_{^pk&s*S*%>DM` zt9$&|=B*WYyLCaG@)ojpuBGPbg!9I1^{n;01@6M|cES0FI`3&~@YZcpLA>z?9%}AU ziqk_)mVDRhd6TO6R@C@49!`K-TI=#AIi^NU z=f=|PC!04q8>?H^4e7__jhU%gg0vG3IM?d$H&&r$Hc4yzl813>O)dIe z-uP8bHhQ?$(uTJqM$edG^VS}`ja*iRH|D3-d@kq@(Z02O{Vs358GQXA(+FAX@)nXU zW3Az>-`|hA+vcqUcx(Tr2X7g*yzaDm$m8->4_&3`AuewjpEASSTa)%V)aI=tc-u5& zs{5X5cFnt#e#RwDe=N;f%qhjw@x~Kjb!v7PqM4JLIkS0Vw_KG&8{Y08-0=f8Z=Jx~ z+Iw^ImZP7VcVNW5*Q}L}w-l4n;-lPo5xZ&;ORgXVr^VS8tjU6$~eNXFw2jw0c`0@3ZPvKu@r=-qz zR!YC&ZR@HBPqcaK3f{UsP=z;5ex4My@BfY~)nZP`qlrs-n%4a$?LspFl9qJWhPR2W z51ecB)(yOE+F6A+e&;D^X{Yql!#K6%VVs(4sp*k4{pHCsqFF_cv{n7k}a z(k^ezKrLmEJl3bc@b>c`9JtZutvh&oY4jcLd&>M)&fM=8_mUzf^Rs`(sgr)vt4>MN z6+5K%UCRl(yfJe`ZFoDgRd4Ix2mDpD9^mcahpO<#9;BTzPR+I4HRJSC(-WeloQ%7? z@vE%PxJGkrJ?KJt+tB^PpSAC|p5Se1i#y%>E%lTn4yAaCeq7#CRH=uYae3nzQI~dV z?ziUu`H8>Uy!8Taw~nj9oAim6CTaCh^=0v0>)V6mXFQuXI;Czk9^bW^ob*sjwes;M zWoF#chPU4=x)c8ZL;3$-=ndY6K3an}X&1in5DtB7Y5e6=sJE^(ozo-r&v-uGSVgG~ zZ}ZOGbhv%L^#N};w3+U{rxv$fca+o{Cm?xPmwv{nnMV_+$GzY9RaR#eYWf+c)_CZb zw3LT&NxQt!$%>3q(-s*W5A(MA` z)9f@yMZ??XS39<_@3(&7ZT;glc+>P&ZywF8T;4RRG&|Mg(Rehq=#g^f+VIxu z{0+@)-ui>Lh3)Qg-&1A&-0v*!YDu#PdSb<0-k4d^?1WnKGfqD>Jt11k>GDP=wNzc> zDXlfUt$%*q4x6_D;BCf~guF?gIPOd1p+A<+#~Z7s=sZPEm$wvEN|Q0X?cMC8Z8mQM z!P}s<5qKm2)I1%=sp*NOnV*{Z!+B$7jb?RUO+utvQcjmQW{_6ph#TI1^6I$7Hg7kA zw;r8l6!V^PVmbDb`|W03>Tuo~YZtzCUEX{%NPf<<3OrJT9tf_N9rVXJRNUV=Z3cnMt|lgo3}yW?Xjl{;?3%<>Zuj!(eqYhS0UU=YhB(X zr|(X!JchRxf8Y2so43K>ZD!Y*#k{A!dr9rr^JY!8(LX7^lW}?T&7kxf-j3SX?|z%N zo59=AyQ=Wk)F-T&kDfP8wiF)UQ;__H^7hAX|ME*VZ?}NAw$D}JEyeVtoYIsef2?@@ z^X9u#X_AHVw(nENO|^L&0^VNgF)RB$m3HAfjUxS;p3A48ai-{8sw>4qdMK2)*Ju8} z3vJ$pg17r-)!7Z&uEl ze;T0iYdjhI^6G(ksLPZ0?Zo$f^Uco=HCoEgxXYXGE|i@Z-hMOZkbP|4MuNA2FIVAB z>W90~^x*5!A|4J)OT#J_@{b7%(UM zJ(YIN9ceuDQ-`c&9$)R=Z<-8Jx6&t6lT-3YwKRUNb$R1gsin-yxZ&;O6T6&c-*2PA z+xCSuc#~?Sm~e`mQvdqc=P=%#PTx_nPl8| zhT*)i6KZzC4yoyp*3v`GJR0rt=39}~GqmCD(1-Rv!scx(c)Rzex$b)!tDjhTxW`yI z%cl@48#}Bw?eZ3@Lu-15xBuPmnHO!|ZUt`>mR8}-w;!u+&7U8A^QZ8;_ghokg=RA8 zc(bY--i})I>wmX-8wcL{Y>B{|RktQ@^>DH-Z@#(}&9C~rq}IP!BgZ?}QB z?KjVJ-&5{F`VN5}YRNCfxt7zWrbp76{9Ic;1>aT7K%F8F^GjNq68mv^<5wjuRm_N+ z`)#l9|K%y0xAEX@(>*nK%gA5z9!s%SnlNi=Jn8N?R;Ol%MQWF~MAZ#%U!VHL={9c@ zz}w2L9=vI$r}4zvZ%8dYZy{AQJC*Wl;*y7T!+FzGDOEJQ4Vro6lQwS?!P~@P^RwSm zcJ8Z{eMY}0HFaIyG^;oPUq9ET&{99)yeYdiynW-Ex$SMTnOYQO|Wsu^`Mo&mwvz8ue_CW3OrddS~H8We<@V2ST-79R~CWE(bJF4(z z0P58n8xiPdw8 zoSNR^^5(k>U%zHYlBZDKreCr8%QkOQ!Q1M0Jb06KB*l5#`+9gH%tOtbQk)*@xV*8t zq*-0k(ptv3mQ!M!+T~4JB~@XZnpH~L+;5$JaE|luJ5K{|_l{lIwBA!`R@UBYzTI(o z^R1r3FHOefjn$=9^eDCA?F-i}oowH4cYwD^t7`BTv7ed@DEAd5Ph8%7rwE9Cd7BR2cHZW}8+SyVniuy+_{hl%>HFp=HnUgvsC)Y|jUEa7VmS%NHht$&K z%;t?;u@g;Z%Wru5`hcFt*u2dIZx2nX!dpGRkKAL3E}ue3hSX~%5~R4)Pd0DTD&H!l zJchR;zB29*o3~luZQ7Gncw-Ma>PGqbkv&K{q_=S1LaIwUN)gY;TaHc)Z!>pxasL0) z+2HNQ4omd!sip_s!@hoLtrTZ|jfZ|cZ_KRGtdc@&dY~tqH)hMwhPNey4*gI2ewzc{ z`b@3CTZ)O*Lry(!_0Vld58=Eis~g_-yZZf?*}TmKZ*Q+Fh&O&N*gsz-$wR;7k>XrS zosT!E3U|ROl4c%i$sZEe^Ttl7HPuUg!`r6&TK&!DZ60`gx${!@J=N6Gc-SZXA(}a< znUis9dR*Sf44sl@W@`EwFH*CIY~JXkmb%UG7~VFVwD&nSZ}Y+1b9YwZO{!H7aZ(Y| zkIS2sO{yrxr96!l&*n|)D#z{&Z-2b=wZ1lQ3&7j5XKL`4!E<3hhe|VHZ`AZlaXoKR z2C0h1AF`JDsimCxc$2b86-$1@+myeJ{I$*7Lhv@X``zw)%32zoqw+>(4ezVQnBJehBW(3|iCF+p=rM13s_Cu}bO{&Oj8V~(N>Oy%t z?&>+t??+k;-rBs7khdbI>RYRP3iZs1v?EQ=^yqn$Dk|gud`~ybc%Oa0Edg&Y^jenv zp0aag-6(Hm>RS5yEp;cU^B2Qgi4()y`XdiL%;s$=c)NE_4c<~uBSn7AXQb)gZ=9I4 zQ^xc0#tc$L%VT(Z?XveCT;XlaK6{^bLbKCPozP=h*J*z$-9zJF+zoN#cqFaA-mIZk zy}W%r{2`B8>Q;(t*6MMSvT1f#>d@f!(eJsC5qOI=nIiMJycJoUwS2kdT8$=YNq22{8@9dE6oq4u3*p}d`Q&@Vk+7C-9n zc0q-=Swrq?y6>s)g1+dh0n_&Y3eu;Q*{JDfJVdiY z=3$(g9%_21UEZWsQk>Z&PbqHhw;?Be^mvE2ODeo=y}t%;oG|N#=oC4nexzFN{U&A5 z#94)!Yo$0nDYVO5tSZ)y3~whie-gj0X4l^jxwOLDqdRNx#@@&_H9fwXd8nC(acX*e zb@>#SjoLR`3P1ay*3_auoHu66sBU=M_lvij=-h9YRd`!I`hNF4wYaVM(-!?wZ}hNd z$uGsZR->i5?)|2zNRO0R>WXV6Pl|Xx-g4~D@OJTqhb?w^yS&2N)Q76@mSWe;$-PLL zeqYU;(pr}{$*GBJDr)ki@Z{qyM<<52?{)m!`yJk{sPNXW#RKmB=36Vpv{U5dzQTD+ z-66XT(dl@zCSZ6we(-Z^9p0K(c-ww!WZtYE8s1_l9x_Sawd^6BH{a^1{oEaE<>QUn zsF~T)hPTfjyn2bl+m#jGRzDJfH>+*~V~rKp%xCxGO`3K--lS|9al>1WYmaa4@OD*& zx4ErXhQFs$57N0yaVbwoT&k7Lo3tvVlOor$|3Z2D_OE*1;qZ2Kg|}f767nWZIK@wD zoG|N3@$xCK6KZCVw3Jzj`{tMGN^vQFp}hU}{VPA~@ODjww@!~Y0&mj97?)0so)DdV zze$;6brma*)F1$ue7vRTBt=f1RH3|$S-$XB z4sX|0czbxV2XCB=bbHgoU2FVYOU<>Crk`5!mrp_ZsuY(pGfpk#(Zrc2MC*BDSFFf5 z&y=2$Ha{=CXWz?D18-~p`Ty5nkKgBT-O~^baa)HFaG+g{Hm>P5=3L(`2?hhPT)5?X#cF zTTAfPZ(0@J_WXMv)+BS^qP}@Fw~BAhbi7&J8s4@{Kjw6sw^rb7$NDO~`R=GG`WyFt zvrZ~H&&;2WH|yLDZ_Cd4$^$lUt-;&6E)Va1@2Tc~Zd~51lZwt8^B2n7E0>-9gw0zU z@V01r72fy*2|K5jJp5$QPtCQ|^rY5$-q;~EJM@intwuAyX2Q(r^2Tg*hUijGbHDBX zy|z1S-fjSIQ=hHETS(oevR2PqtR4As605GuTdd4h9>d$GGz@rtwJCHET5< zDZdoY$D5RmajRRy+vW`)nr8FX9=xr4p$cz&->F%PnjUI;G+J8AI5l%haeAm--lPmt zoY^!U`eSKL-EiJmMbf1m8QzY1_bB`a)be}U0lcm1{fPUXTHGf6=_%Iaee<}y`DSP; z{&3!uJ2kw0tM6H_+V@*W@HTU972cHjqjM;6&Nwyu^o{4^O{&69>P;8Q+w5vDI@>GH;}G6S{7WBJYf zwxiwk-?i_z&fu+2|3}^TlzHkwbCR+8ae0fCx!&@)yd~<;@b=l0AF=*DqZ68S0dG4O z)Zk6xT#9SE3U?lW})Rp0F^KnO;1aCLqyxM(Fc~6FL<$I@T>e554>68BQv>A~!Gt(o*B@g}7zJ9Ky zW*%zye&bg)8R(I8DW`c)H-2nAenC?Czc2IxZ@uoR!kd({vErPM#4zLZOFH}0%~jH_ zS&<&iTKcJ_wNjjGCGGMiIax*0TuVTW|2zW@}E~q-nFCBDHTn*}O@e z`&NoS53cTgw0!D@V5NEI^vD{rIz*>65l;|(@a3mnS>x9Iz~*fLczfr8D!h>a>cD;)?kL}!zJ5)g`FKmwiKg2SPdeVLX&Bz_{ldS` zv3VN^-qyWcg}0F1HI=pbcxx)VW1s1GW0jINyuGjE*RHa8yAixC8S`Ym_tav$;qGD& z8V~&;+P7Bu6r@#BJfy?MTC3+RaMy;nFMj;$-ZpPHfw$2QSK-aKuJz}BRoM%fHuF=5 zu3tZ3kbS=m2X9-acg+wA*I zS{1U>#>(mP7P5wbo zcpH3272fzA0<|;&dZ_7VoSGhu)~uyR(%JVLSA}%NJib~}SMsoy%NxH+EoD>2&3pRA zpSM5P=4~{18~Ah;-jw-kzH9ZIC!4o=>)iKFWB)F1{AxsPc-yz=4~u^d+x3pyk+E% za_;Q8$XZS7e&dv=H4_WxO{&6pNgLkQf4KQKY~F4KZ%dx5!5eqCJL%nnH_eHpeh0?N@AAe^YdyS=?19>M0@=K2GIKK2RwstHH$T;} zi_P0@;H}^6J-nUmvS&@QZa+1eKC^k#WKenxZ-04x-?wev#)G$aHbmepgZa8Wsok3~ z5>hB{KYsG!!8UIbz}vHZp2_!~@*b8ncTG*dCQgr}rJR!A<&7Dd;wOkZd#N|FO#X z_W>t?w;3--;H@d1ntQ)l)2rL>1bq7p=S_LOhPM}o{LT6IohO60Ap@Sx_n!Lhy0QGy zteP%w(vGAF>v{9dY*jS8^&Nk~f7;)_O#yFh7wzV}aVDH+S-nVIC~xQe;b+g-yiEmfD{p!(?t7{^L5(N%i8Sv0 zmU?&0pUqpuvoQDDr|%tpi_P0K@OH=2$h>i<(q5!E*ZOMToGx$7K<(QJ{ULd{mYVrB zI-5652BpXF_C~WcOKsln0B`-aB;-xmXU(4k`DaA2a=N_5$}HuPcELEcl*gAnmp4|S zmMSXahPRII9q_!(+nwO8%`MNzeNUy{e5a)GYdq|Sem!qo#cZi{thz35A)RM*Wq6xC z==sBK-ll`MXYMJ8w~#$bYcuv<^Ul=sCRJ}L@wmKY?Aq}5AD!C&ug%+C;BDE~I^|8n zlH`%%oYn5fo3!g}-lSDhoHMkv;jP)Zw=4tt@P_tfgG=KT}m*UYMX3L0lf zC-tz_!4R2prb)NJ0-)4fh?klSB7V&$?=v+y?Npb1K%csCq)Y7VYj!XT!yd~O& z;qB0`|KAq-J)H&K-q~J-w-o&(`b_2KxqJ#Kc3N*a3*~L&{8^{kyv+u0&y3vAwBFNL z-0tD6Jb_sAEPcw%{nqYNt2WrY%>i$BudKlvZ)uH=_34&sN%DRgp8JZ;+kEiWdsP$WE#3Xb%+#b^ ziqlgMy3qS=OUI{s*}N?PZ(Ccw=-zLdx0{|=Q)NzS-~5`jg|&&O;U5R(q8htCYnxsJsQ7er;;ZhZ=4%-Mz`iYz31UU-?VvK z1m5m{tO{>Z=etupA8(B{0iL*T5BYdw6=ik9+hfZwKiTGOF?gGL!%Oaa%32zo!F<*4 z)qVZGJ!tZmPeJ39s{5{${ESnxt`w(7(%HN*GqseB9_o_c@bl>EY_uEqN*6m3T-kATgewXH+B=fNU zSXz@mn>S5{Sly<`!~U~*V|A{QwB#?v4R0exJh^{`w>A6hecG>^oqp(^GPq5gF=L&c5Gvr(I|B zrrD{)n6=jMcJ`8Pw>!L@Tj6b8w@vPQD&?2rR^5jCt)Aoc&s)f}xi*_OI(c4FoF3|u z-|#l~jG_2-HM@R3J+H#syctz^EAnOGzI^M_uhE=O`4ps8bozG4oXiuV*@JJMe7vy= zt7q)a@OIsIdcDiJ-~LhIZNl?acw;T*SJrK~-y}bCvR0AWz2DdswbYg5VO*o7{8F4b z>EW~)kINe~vy+lGylwu~=vN%x&ad!xORrbm_mnxMUHE$Fr`F`x_?d^AYpGq{_|+n{ zrrVHyvUv;1Y^^oCP2PI?9EZ0HD!jFvQ-!zG6R`ShxDShr*AH*0cU>rNYu5bjXAW-{ zR(N}1V-?;~PoN(1@8K=_1kC-`a^ML=9NsRf@V27wYuWE5kEDIq#`3$oNgZn9?23MVXTmr&*HTL!#;K*X*}O3W zHM3FELtXNi`|a~%hvC<$?)vkPODeogdnG4t?4c=Ym$%frAbZrQ>t^%D>YA=HJchTn zXRe*(+;5jwcpEVA^?#lBl(W+F##KC>deEBQ^t=V`$nbX5M|OPO;q9^tZ*MPZWZpCr z3!FpM@2vE^rKqmSX}wleccIBsC~wmyJvP(f?eYq5&%V~kylEzu!qd3C75OPEl(!R~ z-V6WS$FA?+uBh;~e9&h1JxwtU-<-aFUnaA8OI_W!Tl#%_E0niS%s%~C=YDHm;cdd* zRe0m>sHGcTiu+DR$|J?uo3GC1P0Gejs3m`qajBm|d0TmKXZ-r!UH99S72XED;lZ0U zJzuV*JVnOKry!k?6laINyJk*}W-V%IEj{e5P~Q5UJ{A9*_%7bAs_@of$QJiK)$EA7 z(`ZgW@=G2mF6CkW^e~>yo3x4eK8KmY&w>+$;>ZrND_ zINuM0W;4<;NVO!trk3P!fumE?2|etPnjYpaQWpbUY1an0uY7OY-4)=iQ(<;(`Mr@h?IG23ErN2unKR|r;>4M z)@7X9*F!(GZ%+EDUEcUrYF1C7eQQa6DW1(6xsozSacdfew;}yc$6pXB^VSNyt-F3} z)4JcRJvRK1tci2BE^nF)jpbqIzCC2~=9?{}6T{mt=bqoyeotG2w?(%`=8gSO`|c~X zzj)pnYZ`HR%QzRq+fUB?DgJ|f`F?8y-o`up~Z`Sbs~Kt=$ERL;^uz)?Jw@X-{!47czdMnTaEjk zN_&*{Z9l_h6HC1@y{Yh=*ulc>Yw3cy6`_}UH z>v>~`zEvcDikwm{J#XxaTFPw24R7N|w)}+6TPN_=t>gCa_msU+Tm3}gQb{=Twe-AcI%l_*-|)8l_bq1Fymba|t*1rc&8l0KE9ISBb*(19ugB%hHv|0{ zr%-c8(quF_HJ-Smd zM-yk~)S9)$@K)-?@Ya0Bh%Gj6UBTP(E<3{CQ_Zepd3>iW)h&iMsXF7D9cesL&SH2g zb!d3IYUha4Y~H$ow+Yidc=O#gCs1#?7~WD(Oq0JD-b&|dc>D7cM;>bP)*Za{dUg+Q zXS?ia&avJ;K~g_@-neUODYF$fybbxyb@&HA%76aW1H8T2eP_D&ls!w@>bd53*OH(4 zB@g4}Q_!r^c$itEQ{Um>^932|oI=uPLr|tW#CwO~!WRj!gW z{hD)^@<{QJ{9LQ&jjJ?WS$=cBJ^PPKe`xd88@!F`{dV|!YSnFc6Qwv$NAfUEEzN;( z`l;#3=1p3~43bBRbFHLft!18kyfK5+tra)C{bR(3oSzr^fVV+&Bk*R`ZQ!jQ;$)ls zlutqWDxE2GNL}BxdfozeYIys~M}M=#zTf(Sw-zr&=8c~VNeAw$foJCT&FS86QU+F` zreBKFBXOcx>+7MvP~Nune)ct+w|?O5wSMn}zo(pPQ_*_f8f!-qpT7MR%G)I$y5T!E zZ~ejBqYD~=H-7pw+IPYlfAPFYJJoa>;whB3j~sUL6*g}Jz}wCztnqNI z;94QNOAT+O+WV%mp5i+^-|n?|Muw#qtCW^8wlQ}+<5(g{Vr|zj;P!l zb8;8V-_*69H%>#+nrVcr)#R5v;k-$mGhV80c$?L{Rc1|<p8JdbeE^isTHoTqO{On6?-fjkOQ{VL9?OOa~ zL2Z5CSo5sevu3TYN8^vn8!J*vJC!{2Q`7IOvw4%U@qDS-m8A`DpE+s7du-lrA#X!l z?jG-{wfCCL$K}npdXfHY-XeBlcV7Vn->iHg3a4-@b<_9jli2`N}QhbQqbe8 zn=Wtpc1r4eGbsJ$e*5|Ru3TgDHUhlOeY+8OQ}*BRU68L|!}A{A+zDvrmT?w_w;rE9 zu(!?INbojdOzU*-Df_3+=s7oIzB%1#Ycg;b)LbjY=_yk4W-IHE4l~vo-j@IJl-q6I zMuE2*9(4R0@wzvo9bZ?}TCUTto0-&3isW|A6@l)uP$HgB3v8f&M%bqnR~ z>nHV}W%D);yltIOgSU`Rx^!x>a)#t_d1E$7Yj)(zgQgZeQV)#NpUoS)V=ceh|X1mDdd=H^iNjm9$h1>)5-Y!+ zH>si&x29)!d;9M@THCx$1aH08M&M1^kJrs0<&@&wJ3Z9&luv9k|);MLV0_w`{Y|~-ll-JnNQ{9jnmLfm_KV%Gf#+?a!S3GPeEE0 z(uuTI>O}H0&b6AJ>B;7ePF9q3hTpuWAMf^&Z`!=w4&Fv}ZJ+NwWk2kJnp5KMx2K1B zs5Kt?S&MN=XWwtEPAz3CGR}Ue*#qM)Z~Q7XGvv^Qw~uX{w7<>URPfgKE)U)!_SW$I z7;{o<_A4>QJk-pY%^NdI8JNviOF1=h$s_fU&6~7JinBU($!~aj&tdzmvU!^Z-a0(z z!JE`WNSr4dlE-&#HgC+9qC=h(HS5OGE^o0iv-6TRybb#NyXM%u-2vV<_vjG*p8EEm z;skwr(DUY-x!(Ngc(cyY@b$Bg-`F8FI|+$%Ej9CJ^F}8#`07#~!`sc5@7QVcb{BYC(x+qb?5R!3wG<9hik)m zW44S=3~xVcb@V`+x0&Fr^UGCu)8yxUOFuPxW1O0K=$AD8)XXWZW!&XWlYtePjauWO zU!$d*QqT13dE+WgSC+@{_WDVuHn(}31>WB1-^qPXrCnJ08}1WdKl6mFWgd-YEtfY< zhNkFodE*>n?WmN~@V51Yo)c}}W`nm?3#;%JtA~2d6PLFXC+IspsRx%g-^@zC;q9Ec z@9SgpHV3@Tc(smr{3_+|=29 zPm7#Jidq>nY2Y&-Z<@@S9mUF{sTG&EjEaW0W}`+gvw52b-r6j!!dsEk$mqX;jTHUl z<4x18uSc^xjYq0mC~q^*deiy+Gz}vkycZvI+@@Hylo;%~z^u*H4&l#3afnSv>(nBrz8D~XJoSs-ZA8)?Z zOOr9YZCd*Dw`|@Pg15Ww$;q2EN#8zw{jut%yWgY^rMT||eEr4nR@$B6?S20>*!lgI zi@@8UtscBd6O-b;lcC@DCgECYDNj1ySUp6COqFZ%@y59^TgHwIZ`;OA`>K7vEe3C` zhINhmo@VsZ@cn;1c*1#8oH9Mk*`CZ=lRcRIDN^N-S@q?Euxb3xepS@4}b+e`5ZPE6eyh%MP zd#m}W56Q1tTg?5Y=_F)gTr1UbdE+WcOW7oUDQ8DQUv8?N~ zKb6jW_{eU1p3`vjG7h2Y2dhYbKHPG2XmESG`Re0vzm<7AqrzL~mAeUVz8}YYjQLi{ zaca@|ER?seesuk#4sT~xczgZrI^vB#5>cl(wS2ruov>SK$c+C#%$D524&pvHt!cFpXcy)UWK=gkLBb|*+Zkhx$Kh^W%I^WA(|CK z@`S8)c?-#wvDWZ5@UTzfuOsbM{_~K3RCs%}P0#H2)OTVUzs8fXFE56rT6*53489%4 z%HvzNP~Lt&zuh04`|bP+Zx2kY!5eoM>#ut^*5tVtJ#Spq6m_Azt==-&{qx=nD!fg5 z!h<(y+Wa1XTJookvj<6+Pk~=e-IeqyV-M7^`YD#TZ(sPJbH80!;cZ0wUfJ)dwCfa; ziIr26r+D6^NlJ0eE~K@Lhw~mwhngNqleDsyoXjj~ zX>BQPc$<3i$R!SMS5$braYhZ^GV(XvKbqd?VNS*+O~0hMR^!q0#>^=+yOr|OLruRF z59dwFY*jbB4SD0fpF6xYukbeL`6|3wc^bH4T}ex88K;)=Fit;ph{xqk%Eq{_mbzsg zdZ_84)_C|j=y~(4!cI!s@HVm4@A2zHcK!F$D=WP9?bSE?J!KEEG^Ziulsr;g%ELI< zmQR6SO|4lybuH%4#~U*!wc%~_tZP2%+;3M^czbJh4c^Ew>oU$sGEOb6W!zW$=4YQS zZ~UrnC%%5EbKgAdnK`M$d1JPW4h?Uo4ExZB9Nw<3@b<#S8oXuXulde$8d4sOM_L;a zcX|t@8iNTvOp~dEb7;zo!{JH!$owRbPKLZ@$^MqmZ@C z6HB|i`F5!E8{Ss_dGZ;~{dR4IxAF7ql($&ZmhxotCar3UahJD{U1YFlcssoN3%_xA zyRO1p?^ir{liqR0Gx}+`yQKXxkL1zhXC7+i%;rs6#SHbJHGO9Drpci3l%`>L+wt+P z?*nh<_c{Dev+MEu99j?TU!Mf_7?E_HQHe$7WqlP4QEO$MdM0QccD=eDf?SLUq+ zc-ye325+%;t?aqs5=>n;E^n#3^_?F5>3FktZFn2EZpAj6x0c}T-q&jI#?F-&YXdW} z)-q2l9hbL|$@uQbw^m%8sNUC7PnRT;gmFfde}2HJ=Clv zY5JK{@@MlVWz%G4Mds1O>G7?_wXB=X8=cG`=?uT&t^XlkxXI?NHF&%6?kc=Vwdy&Z zF^8(3mu%kZd8fX;W%K5nEyHhkTk-Jb%{Fgsz+1~Vs_@277By>0=PSjzHkOupV}8lg zba~^BG}Cr@V+Lx@Er&L|o$$f^Pq%ry0ld9*%Ru)%%~7lB9a7JEnl5j?Q+0Xs&7kxf z-tOzV>I*h+ZNb|^%Omi{PZ_nc&nT=)b*15i4o zE(KR1wWR*pXTE7OGc_|yamhnJHT@yeDD-~&_g{Q%rp;S>@HTw-jp^P~X?jwe6V_-~{=%jhS{yJ_t) zcTShLrqa3Z1oH9bTczYTyd8b|mCnzn9l_g{9g%s<(PwUESc|k!OCFauI(H{sC~rTW zd;UoKJ?#YE9vgL2_It{ERiia0CFPOU`udqiT3bE^>8s2@O}}qkvo^%TeyH`lvARaH zN=cjh?VfWs{K)35GkJTk25(l*hPz0*LnJ@r+=~>ahnjwBUr+Y^#%$EAC~056lt+p) zKQ&K;aXoLW$PCmO9>d#;HDCIU&081nw&MCh+4mds$I|SdT9aSn=j5fF*}QQTD^k-# zO^-%%Z3^x3#;#aViqn%ZTf^I_ho66&&0AOSHuu&>;Enyq(iwATxRY5+&l^`YMeXvI zXcvaJ*}s_n6Pveg;BDk1HF)zqN6lVRig*HZ@RompG0W_h4OaOU9aO8I+XvswNO9&5(dAR{t)lUh z2WqaRmUcl8HT{gU7Ii+}n3)x+Eswe1_W#_dN%sBL3%t#3dvkrhr`%VF=H!`2f4@l? zm@Sr0QA^KTV0FXWhsJzmhRs`V@HTAnZqA!Tf}S@iLsN;nylJ@7c&rH+-p=cHmGkFs zeZX7$H6FZadeC^RKBIg;=9{y83iaHrZ*P*{<;{0iQf4cU;q6-wKeyAq-}-{L4IOTA z-&5;duE~S1-|Dkwe)?I9TH~SLz2Ep%X7klW=F!x0dE+W-?o^7?LtXM4-j3Y-z%Ojx z`hmA~Q>*aCJiDuQd28xB9yC~p_ve$)n=w}IfT*YrB&jnhlP zUen?&WCDfqwsO_2&i8LOg15Jx@!(DKJ*4E(Oibe;hrXJfG>l z%3B9-v3VN=-j>g-!JAZzyQ7Di9$y_Rk5o4wL}q_QQd9eagPy zhJ&|`oA&T_wtn)~gd1*b&539{QhvsL^SJk$Zw6&`!`mOeckM)*w-Mm&rGX>zy{GI! z(%1I8wBbakCBGEspDSweX#6f?%uFqHl{(HIq#e;C_2cp;IT>eWYI-bfczgACdk?aC z8wuWCUYw9OtDgq;qKa_gnT(NnN7(8{S6Te&m1Jyo~{Gw=b*0TdaPh3B<}DlBYg-zA52zt3STc=S|aL>Ff+|laAct{P+4v;BDpTG5Ow8O%L_#Nyi&MHPnqY zjbeB!?auJ_*OASg|37Ooc)RnVM&`}>Zz5IiE_z~}T0Y)b|E^=iGHHEq$Nhn_tQ!#mlF_ud*WJlBSn`A$swbW3sH{Mo$uX3Ov!-n#es z!Ov{o?gVeQKVF45iD4<8(L=-cSJGO>sY7ydt&}tSeq#nnGc&cXpKGbf8{?Y%^tinF zW}`nt8{P&T{W0g?Gnx+GCbb*qzNhS#yJkGKmg-7z-+r?1H{Wda?00!f)UDxdukkmG zwePpPz+2xbHF)#wjr@}hYE4e&p==}fnv%uSeX^p@e zrx9`rA#1aFOWk4W{G3doybb@^k1nzAx7pxr^iz$%8>dmE<|$|M7ON{(jFr7ey){tI^NiANt>S+-aX~ZpRjqG2i_iewhC|j)JVHvoPJ-e z$wQAuYkF{b<0?&+5D!l(B#&mT%Ui6ST6qj_XRkWm`TM=|!Q1j46Y{;MMNUjJRbP+G zn{NimA2JzftrVAfVBF=6U)5yRcuF}9Z#=_L z<*06W>)Y$hqix<6gSS`a)+uk&^nB+a`OBw}x{6et@zg!Iyd|n=c>CX9wwP@5wgkMb zd?_byiF&SjH>Ri+_kJTkl1{M;O)Zx8DQUv8?N~Kb7v1feWkPmg8H+hHFV$ ztBG?+^fRu}(ptvLhro!GS>xea$(;=vp5SZYmglAm#3&AMF6I5j;k zZ~Q7XGf0|#YRNCfB_>01lC-i8$;J+?JchSr$M)^u@OE~Ew}&=+@Wxuc6Qf^~hyE1W zG~r^4Ixo2SHmPgCs0`W!Xr_{KH;Wb-Cf)MU0ihPVEoS@C;^w~H#gEm~QHH|hSAdXwVJ zLv8ihaDUQ6O}`YUhnjwBmp6Wun%SsBJe-)6hw%`d&6{RNQgv&s;q96!6VG>eyST#J zymzYb#yqh!d-lyKt!11i$o%v$p3NJbA)3{xrJURa{nT8`I5jJ=a4#A8)BABjuOk(j@86=1t0=iL*+{Z|=AMdGRA7 z9o{ah@b*s2JL2ANRu2t)GOqa?N&96U$)kz8yoGe7sTktnu6_H=#~UkZIxlr)csu)2ylokMET7@s?w!hPU&^-TW(uw<{{VEqknv zc+0WJhWpcZt;?J5IZ72{<&pCH#tY?bQ0H$x?C{pS!rO!!?u`4M`c5O3-}jVV-ePr9 zZ+Qyk?TGhpebV9W$_j6NCKbe+FAu3tZ4YnR6YxE+lHa_iZTCLMnjUJ)V|aV|@9%d1 zd)9Ro-UfBP%Y9FoCxs@@De`M_x|8Rsdgze-_-gi_&6{uL48P&+_boObkNa)yKmY&w z>+$;>THaX@a2$c8Gd82S(7im_z_Al{%Q&^hL%&AH%9#(i9MuhQ8@_wY*DAo3d20dQ z)<0bkZ<+}g>Cy9+x~m)~o8yy3>Jsh9@YeZ_58{F=^VSl)E$KEx|DZ~fksftvU-U~_ zQ%mw_>K4NrcgL<6mv*7?FsGzRTKKM|ZcEh-Z&x?_+^6mPtrd8iG@}Y{898fyvN$Db zPJ?l3dQ$81Q2{v!7!P~IstMKN# zyI6kd)MDjlp72jPv(>YX%Ui}y4R0&Ha~}QxuKb?10dF09&D6iAQZ0+~ns=a-Cv}{C zX5VkAJ1H`MHgEilb3U35=`U%++s0$w{Goln-2mR+oE?ETO>gvYkBk?o_uOy!CXjJ< zhPM-jjQF+9TU+q9dSfH-RwP$at$fp#vek1u`+keqx#8`h{g>k}h?L*ccHnJ(-&x`B zsix24f}p;58lSiFNik{lc_yqK6~19rtZo&zi)4a-fzEa z{^vt%-a3G{F0WMKO|v6WX!J`;q*G^7gJaJ*>ib zNt^fdkQtOzE*`dW&Xlg^6f|C&&Qi|rcy;;zmz8=p3PfGW^1kC?cUF=_@>QU z7w|T2&>Z(YWuAJN82gE(^YIp|x|AoywDa+nqZ7m1wzaeI3suX19?})O^|-qRZ_;F% zdb~b)lct*E^c!BEe7vy}Wp%^bj}|Z5X5Vk!z}wa>k$F@0)-Vn3OVYkMHU9D`NKPp( zby&~waNd-=GrVp6?%fM*-nxUg)wj&eeovLPy!feyod3$aCHgcO-Zp&t8{KT)dVsgt z%Omh6^{@H4TVjuEsU<(-8h@ELuA*iJjpo`A&010(dNexU{U&A3=*sYR+SCr;wt4Fb z-llD-|-DlX~Ep(vzb%ukZJK^RrKC_N?d4x4N&NU1>CTBx&Z!$D5Q{ikB)H-X8hZ z1n0lk_W^I)cX;r|U1)Ub$!mI$JbK=w3{54TkGCAVGraA4#Gy;<`>ii{TQ_RH`<~{g z<#k`xL;gLyWltc-U14}z_3fcMZQlBUwd#b zrmY`h^EME?b$+A@ZyNqtOY-~1Gxk#T{$vlDwd|*O-dLTTQ%fGr?wE%u&^aTU#yEzo%CInhd1&^AxBxb+dW%%|=eBHJxZY%t@{3Gn+SN({w_Q zwbtk3aS?o3}yW?XlHWc#}RgQk=a>TGO+XpKE>d)6YERQ{Y!M)tP~7sp*llFN zEu?d+y5X(Y%t>Fh@3))5+t|rDc_Z8GS&Fka$-_L<^ixY7#wA@o1?j6&70J&y*HY8t ztC@$Id8D|Wx6Arn>h(Q2&$QHu;ceo$rORyIZUJut*5u@kJC(H6kJnzLoU!8BcUr6} z^^ix;TVS__x8oOeeaz-<2zYy^!(#V6v{Z|Jdi1@H)?vQeLeJ3OF8K&pMu88RT?c# zNwZe+FsEFuNjT2-3-AU{3H%>#+zLQ}d-?hH} zY~Fmc(a&9!wBhZTkGFrp=4~W+8~JP%-k39mw)$+8H(JkIiU|~%GaYZ-#V+24_5AKn zZQe$Kw~jrQy6-9T)Pp9UvHB^WLW;~%hpB5x`LlVGR!O_ah@1QEmq%WEsLk7G@b>ad z58kZzy5U}8JfoimzvQ`A`13|7&brj}Q`6(#Z~Us%A>+&}#p$7@U(!-PQXcxLvw720 zXEw=SiW}Yzd2OEa_us~Vx6Lnj@FvxgJ|j|G$`cazt>y9-Qr&m0Z@0er>DTiX*p=b! zV_!VrQTu)y3*J`szB}K08rWOSyN(k}q4m5q#htR}LU}vvb0-{S^L8tEyKAloZ@gEj zQ}B>FfB6)6E-7{qk~5??PA{9ckSf;h%>8!F)MJO+yp02I!(Q^>&C1{CcT`9}`uk0) z67i&RPg2uEHgD4Ia>Na9cYOc%@3ncm4ZOAQzbxN-s_93XhHpHimb6ySo7An8x!&S> z-ndh0<&F$**ZqCyxi)X(!P^T9s_@3!m|9aeN1wSr-BLeN&x~jDMyFIoic4!Xae5@p zx{`-+mp6WunrFv2D^gn?!`lVl9$&$2ukzowoB-a|ZK}c>bNXubOzoQ|wLhD;6xBHm zYN?jSBdulJ<&9sBRi%{2@V5HmKA)-pSLSUZc$<6UJ?Y+4?n~0X`}Orpb&Kas>LhhM zo41J74R0qM{a-C?-X?*!kxM*yv+hr?cQ9lvXTm@2q^3usUEWACyY;Oit@X`AKQ(v9 zTGaI9<4vl_xK+{c_S@qZO|*HN4Bkd7kZtxxCRS zWner+OSLpPeb>6YNf{WY)^t_!7~Z-soixbiZ3=iBbo27?_f&K4%AURM%#fTeZz0)2 zCJ?fgJxh69-nc5FHoUFqu)_KKA-99KmiI*9P1Cb94b57~6Oyxh3LzO%bS34K;+p=8 z;jPq(;ce%lyF1wT+f?xO^qU3oCQU3>Tr+vrihI8?b7N^~0>$uF+O^^Bl@lLaWb-x+ zye%DiZ}@xKSQ8GZTc5n~yC&-Josv3clPIM}sSR)QTaD~v^L7V#n|NOh-eT=q>Y4ee zCBG)F$&<~Sl!2!sc{Cj|kL1^Q*aJ1!lCT++UNDUTEnpFBI1v{b#wcsOs$ZVhj({_ywT+PqB%Z(T;-m+n2~DJXmP z`uUXk`R78^QeBrfW}{{X#;NH^p`}`kbJvU)%G-6@_I}9b?Jn@vW@SR&QcONpPM5b> znKgO%-f8^w_^x$%V+Lwg$)U~t_POhp@348B0p4DCdpG2bpKeK~-``j#=Tprzxi7o0F#&G7~9J`4ptDhExn$8>=7YDU`P_bRYAg z&D$*SHv8c!yoF3mvsPkHifeLeJk0OjZ~UqzTMAE#9t!1c&E6-@v3Z*f-fn4mzk9!> zm}HSTHK*qCmbybtR~nCRpM~-^>F}pU+PuvHZ*6XC1m1YtQ~OSmex8(+lkxH?Fv4t- zrk|RA#;NHE(Naz+?(!x%rMRZs(pvNV+t>rQTxat(7redoXk^}a?(CC$p{9r0*W>GV z?>A?hoP}4&#d3@u{NzFXeE^nGul813i8{WRTaKDRf-sXX~*V{ah{hmtvShcGD z^u%3LYu0K!Avs;%G!^LynGEyzYR#_c$>zgX{ zxEl()Bjb;UV0ZwtWNtS2JyChb(3OssfF-EiJQ zs&fLd@-Qd0%Uhy~=6?Is@QJHz-WGzlTiUP8eoxuCq+|83$-|d|{YY_osKa?jfb@)osT!}lv=7% z5Akf?*rD%i=`U%++aX&fJ!#)>i^1C~Yisbves(9FkGEJ8;7P^GDdo}gCS_2@4R76N zzPp*t+Y<2haHj{u-&18jRe!$FOkU&h?Lp65it6k(b^d(3NnJ_t(vA#oPhHvkdp2)N z!P~Sus_>?HDPnmzd1`52tm~_p-{p8DQUv8?N~Kb7vA2~StSjp;R7gN7!L)qcZuD8;3m?1yU^FCPNG zN-bsfjcan!L(N)@Q*%f3Q^&=PnJsN_+r0Ewhg5LetIXRO72XDSd&qrGxg$wiwHnx9 zoNF1U)_CZr_FYTAd%ZC;HCIV-ddL9dT&vN(YbAd+Z(K#qT~O0wX~WwO$DD${jB9h1DO|y#}9>d#NmwgKV z`E=L)c6Nogtvw%h-_snma^GW(RVy8D(vHX>wd9W#&&OMi4h?T_jQMw5Z@YLqr^4Ig zvzq{KIrd(af9A>NjZV$E`}#FGV|nuNrm4xZuh^pPpCYZEW9Fd*0LH z+2cv2*7GCIaS}&DW5_;cj&t#$zLdMul((VUpTy7T;Z+#phw;JRPz(J zw7ZZv>uU1QPc5ybC*A$VDpDtm(@!mV7^jxj7Q<(q&s&r;m*Hv8kvdpW#aQQ@ucww%0W)NS;;OX`grMxS=0@0z>g1bDM0-L>Iu z(vOC;b$DxD;cdsT)p6fb?o`tJewAACOL1w!j59xVdGd@%Rbs`ZTE20qA1O~<-lQs} zxZ&-f2lhMN;qA%_Z!fKA1m2{HkrT{Mh4sX{~cC~jSb(#(QNPa1vjyJ2K z;q5c~9{j4q+jSM*){K2(_kK^UnKYc*?svas@08R@&X9QN(_wfU@|{=t_e0FTCw@}1 z>+$;>7Olz&obShvKS)!DY%*kRHgN2aTB=A7wd5}{PSmn@Ep?^bkpZsDw!=PIx!}sY zwE%CETRpkwg43UZ^eL0#(g_wB59dwO?XC;%*?T^}!RD_-F8;{lCjdv+Ed+^oFqtU+kHU4bg*cG!$ zae5@pwbb-8PEC)m*7N3@+439S&R@5xz0F%|@V4QGHQDbecTG*ssjVIwzCqW-nP1}% z@wmLPIyI|Eae5@pwbYVdiu+DGoHtFkC6D24^e2Z+w|Q#=-X5J)gEwi{Qe4xUw3cz^ zp{7TprPFbFV>Yg$rYA&8Iiy_U2zCHm=MJkfr=UMrV3<*!9yAB%8EWnuc$Ol3&Wt zeyH>DCRLZ>rHY2Pci(<@XZwEZ3f`W6z6x(r&U%Y;s??f39Lw+v5F>cmDjX8+d!D*Hi9$s?6VTk2OVq`4m!gu3;cWT`6ZaZ_=t9al_kh zXMTKveZO@FZ!_jZ=8c~bYEA!=pYd3ynU1%Rood$ldJ5%j=j3DXKiHT5JfsJB8}(wH z@iXKO43sG5WkdDiWkb;1)n|;{{usrx1Qjwf4`^qyr<#pNoQRT@j`i< z`t(;WwBOTS;H|}iM&^z8Sgh00{)`~4l{`{h%ENe}yv_N-!C$g@ z>j&QYyj~D*(&s{ob7CoU@w}zjg(hcQ-ZCnh`|a^3{xH+#tv`6%G3eRw_cWu2hCdVb zG+*bp`AvyKDY0h2a zVYj}1uJzR$vpTgVbCDj+ zv=xI*_Bkq@*Cb>I^%;&Y~F@|x3QyNaNkoaPs4rX`+tThy?OU<5*6{XkU%jfO&D&7$*7u<*y!qA&@k^&5 z#Y1XwZ9d*ocf}54>1^I&RVn2$ymkGb`wzEy8wTEXT))A+-+X(}d@ksr_N^7mpUoRP zq?W43@<{omxTdbln`TvxiiWpe9)7g*`z?oqx0h~>z?;-ttayrBC7ZWco!3Jim$yV+ z8Q%Wy+(X{7@3#@)?ZHR(@K&e)0E0w@@29U&-fo`v)%V)GjRbGgTW<`1Po>GEj{8oY z{yq0w^a+^z?fzpvveV{m6nMLNLImD;jwu-P%`f>Gk3Q{&pW<01#Wa{xs>OJr_uIC^ z|LFYv-qGOg`o|;i#tHapPKMeykFP&2Z$);VZ>OYAlOe~B%>6d}<=5~JXqEpy(irge zRNEJ`-_sm*Ykty0a%yI!@nrK>4;^ZDq4^V0l9s(|siJZkhPNyJ{>s7jdpZ`p&7WL@ zH>n54`SUfk#zQ}~?^^mL&HUxbGosN_W+~2`k|!k2{M5{u&6~7}88n`f-|)8aqV_l0 zyxj`kX06G|o3e+7yN>=pZ!T!exe(8$IM(S#WfS)T8*C`uFd9+t2COsv;2m) z!+)^g1e>>U;B9orm)!R>BWJ^ety)RrDf-Fgjk`-xJw;CTOdXduX1289ZQl0>JO91@ zHt^PIS{2@`JV{w_--S;#{ttclFaql(zNq4^$xeLunW%I@ift?%P zj=Slq$86pvg15V-N9K*+sd6vWl1GYjEw$ulT$*8A-dK?xYBbkUbFCDoN762DA=#uV zr8yhk&OhM688&Z|z}uK-tMDeBkyNWG#>06FnUb_tiib?tt#&>4^%I0kfcxyGQ25-t*HNRi7daLU9trg;T?>9|0X&2I3-?)@tifi&`^Co3x zT%${!7~YQBw^>J@RC=-ll@L^}RRg-_wj*Rqv~M_Gj}JIQhDohMu>;^D?}BtHYO{uz8yX-j>eY!&{X< ze+!&^Zr(JUYdq}V<;^z(vt{Ui-fyG2U1ali2Y9>fr7FBh-xaWy#zTK<&3-sf#@R#p z6!=wYIyIVWsWofq;aX~XG@27|d6P0|D$+w;%4vA}%bDj-vw6D{yxr3O)$sR}yOZ|9 zxUZjUW9jlKFk6vb#j2&rqvws&$e|5yXFdAD<2G;8!CUKvRe00vB1b>DUm4$8n*NLD zP4jove0Qqx$K@?!wuZM=mp!@B=It)<_ROX#ylHlkQMWLzG=0*;p3A4e2%XgQ_%g$_ z%o(CJIr-AX~XThotwziC!AMNgr;J$cdg zPWJsa6TIEJq)vH@#g#-zF}$VTg|s`y3+3(dUO)ey&D$*S)?;&>^2Uj!(0h0*Pk^5Y z>KrpS@9EViKCsm0Z8ms&bMWi#dn(oCcLh>E8jrMAil@jK&Ra-VzH6luk>Z@B{(cMW z%J6nohcmuw^EL;(tzG898+Q>)bC1-%`MH)_$`j6;l$mi&C-hT?tYscOZ(QZ8OWhjY zKKP+0+S$C#1#kDhS%o*&inVJgC-d{u%DA*v6PG;Uyh$BO73rs@pPJKPoLb7mxXT;A zO3loY&hQ)FzI)|D=li#L;BCRs&F*_D)e4Dodek8~L)M1#7E)cBfD{j@1e{aj0}$)oWwC$*lpkSeU$SlZ<+Q78Z8?cU`l z+q^9RZ*8{M;Elaq+wanbKTo5khxaWtJ(8xMntsOF12z4U*7GK1V4PhsPVLJrb5b)W zwd8SmlbnoGvr2|GynS!pyBFKMEd*~ZMs6wQJJ<#TD5qNvE*Xnum&1`jR zcpG`bE1hlL7K69P-uB>)Zx%J{GEPko_eD*QulCJTJ_W9#)>PN{V|k>0H2Jf6lQJ-_ z(WMRzZB%d z(kdw~bybQR-de9Z^g8=~TMFJDc(@90tW|HCuNbw|r{pQ0g0!ls#HHOap3NJbfx9-m zegBjv52*0AW}m%JJGt5Er%vdxtn0KtmA+4(-11HLdSls$?G*N7fu`r(12>+Oq_c68 zR^^Br+#Wt|KRl;tSu7_gm*5{CK*<+gTOfo^P`?`#t5vcBj+Pf4cb%vZOhs5Y7CYHsike z!+B$MYO51-zYTf#kV_rj&aUwG&_oa3nBR9A^hNqi&*AO7 z3U94$ufZGdUh2T!yx2%lD=u$RhY~+UPO?zmUi!r`mpi=uqr%&ZYisZ(O|q%Si{Y&v zrljXh`gACFVeYqM@4sw-!`t~4-d1*cEBieS?4i-`F=ipcbo;R+_p$%_G z9ICpP0CoTlgj3;-gfQV52sWpZ(m)orn|%2g%#cgJza%2b{p$E z4NV^Y&+F`gnv-WvYI>-dC!04qnL$&<*RRPb^}||{c6sy7ru3WpZSME>8SC(NQH8fw z-L_}Hr_wH@xU#oKzXP**lXh3nacS}{Z`=h}QPZQ*rL~5)l?R_Q+Trcu3U5!}6`40p zZ}s3w$D8KVG@e*f%H}Orm0i63+aCwN=J0k&g|{WodGMy0lE&k^N6m}lo3nfh?3S9@ zSe+im>8FSB5Ur^t)z$MRWngv5Q;M7WZQj=HUw3%Bw8Goyo;%{cry)}>N$^|x4>N(-rl&eAO5-B zU4I|x@(OQTHdNs)q%QAXNi)C3Lw|^7t?>Je*(A+wHJ*@9y0q5ijh(PVNlSjlOWN>u z!0-k8JNMfa72cljvorfWWlpId#wEWL=UPc?)=JZ6PUb0}0wZ)vntp2f8K;&!j7wUR zhaOHNE^n-2X~WwOAOBEShqvYx-qy~m!J9Mz#<@GjQ)sCc_b%0?r+f;MlX0$+s?$%+ zwUVYkmS!!NH%%30(D+M}F}yu{z!ms)_`BZID=WM$dpQDcQa`?@BjrgE*YlR5x|B1; z#9ZD&s${G+yd8S%kde;)c2$M9@dMt@eov*zNb!t*lKvdwTem)WlcpLHcX$<%0tI{gTL%*cCmfF|PwHlqx zo0LJ*mE|$K?eomg`yAe`sqohMwHmxJC(oLF(i2Ow7Imz;R7u-k%!aJ|! zlR@JtP0#T5%cr0Dl*8M#72dYo^iIC_l&3?j?5*nErO882h<5KccFxRw7Q7>LDL*IjS4p zzJ1>Kcfi})fBygV*W>p&JhCMxa3SN9)=F{T&1>?wz-dB0w+OnYP!AK+^q`puJ^6OZ44SU$!4uA#auAH zZ#ONk!dpE|HFXazZ>cLPPc6!~Biu_kZ_Jjl3&Y#U(@yw~&08z*_RiKSyk+E!GL3r3 z@A8(SE9FTgooYC5%E=hs9{kj_K{ju#!P_&#TDb2ie>kC*>e3@km>%X~T+;MQnrrEi z;(Fes46Nd-Ii(c&*-tia?1Zb>6*WE7C6D24*fX!Z-{!3icza|;72cSKTI!9SdehRZ z^t{#cjx;Bd&0ECX8Q%6k{k+fHyxjoaX7BLeEn;s;Ul2{7C-^h7dh(%$!3T-k$r~fh}y_+JU!@4|?#X z*`4H(;@p?ULw|^l%Uek2TuU9YJFd; z(?c!!8Ta)U!&|9C!`pqw{!ee4w~pX#?5Y~Pai@)?r3p)MO`maj(^PCMPd?sq%+~Ps z{J|F-ZS&R%ymf8W+P&YTKSPk>zLR7gYUW{_njT-xJmphhgqoQdr z1x+U*RW$iidtBZURW!UEe%L22uzBkW-ZtLQ#(hr{^;4D4#;WD=rs0bB6+4&W^fZ<( zl()@izjCb2TQ~6b;G`P7Nt0w;qcx|=6Jbuqsp+BCcwFANN~1LunTP%qnzg9&@x~6B zO^TO1=6?I=v?+Jkymbd}_di*KH`bCgd!VMDacarKc!*{#YL_>Dm0GG!k4F3EpPS%pNuV2dJ8xN`L^5$Dz@`rRK z<;Q>*Znk+b1@FpKSmxU`mjjYpau*D@zHJ(AAH zo2DzNB0Hp}pKd4o3~!zZPOndkuJzTdC2390 z@+nA8W~N6IkL6)qjh5zx_drwF&q>_-{LqZRbP*Q-?Dg(7@65bspO}AmzRx<}S!>Oj&z?C6NzSlN*SYU0bMCG>-~GmENS`Kp>Otq@ zEl1~ux9_!o?+I2>$ z@7h-Xp0WpO_QNM;jP1> zH)h%QTVL>Y-`py^rI@7DKPMx3q&U}7OMdRFd_z}vhhtMKMqtFipjNoo4YcfV;e*PF-XP19kH*&5!qzHP=gZQlBW zx8Xh8<$F)rGe0kqrk^^;z8b!tWFB9w=Z!m(s%SjY+K{-WZZ>b43>r^q7lyaBKRK?s z&D#L*)_q}4-q-_~VVsj>T+;N1X!gTAjF(S=ozNMgrJSrTd8E4ZQ%iovvw5S_SF@9n zHoX1#kZ1bXybT0zEuZz^jT7_jErs8=ANuw8TS(`usL7-8(?c!gPsf{8-SD>Lm=}+< zc^d@Y9`19!{ymj;6cV?3uK8B_`la5od5cv=s$OI~o41HNHN1WG&|?PMybT6#OPAE( zjdx~>DKS5FNG&N3<1TOfs-)Q!J&aTHG&O#%_0`Nzoz0tX70F+!Xn0%won~LRc^d-W zZrW0VHz|KT#Ce*&{kXjOW@sw@Vt6Y}!|*ocjH{O0ybT3!g9o&4TJNcJ%FXbv*wG^~ zna!KDikX{=*7Fv!>(W}oTl?hoNLrI$TFW@|NLupid1D4@?wXpOk~X|;K7ZK7HgChhTkEY=c;neI zrxd40qhsaK)RH{e_nWk;$at(Nh4ZG|wc%~Q&wUy{0JZ%8Pe*{aCkJ4>FUMO!9 zPM-8XHg7k8x2Zd7@W%6{_C1A=wZ649{f1$i>xoO=&_Wd>*yuET;72bSz7mESkx*`5OyhWdY`SrrZ zXLed+^EL*&ZF;2&ZxTOJJmlO{td;s?+?_VRN}Zy^6gj1y!|yj{rsgh6al_l+-v7=1 zHg99W+x#0kx$i0S_-d&i-#n@PE^nzTHdg*{-jtmi-hO)UC(gBb8wcKQxGOSm%HC@J zfBDpT_V5;c0*1Gpw=Qu0KHzxp*71!x_`~h^ zfox3^;Mx754VElFygkwPy-Vyde>QJ)YAVtbvX*%?TG}bsy1a#Sn6cLIws!EdH`u&Q25-|RyuI|qKM%2an*!dtweA}Jo+|mT%B1g! z#Pa9kjXR~zaXM9}!92b@&Uq4~Z@yj8&nk>-G}lsdttL*7d%rQ8q?tkEDfta=zd7_?=kGhu z25%2Nl9M-O54k^;+%NAy)}=?%^i!8l!8bEANELnKnw%PsRM+KAaYf-bVM*FUn{8C)gL-`a!GH9yQgGbL>V7G?1TNb?kE}OSG z;BD^B9=u8Y@YJaJ8PTkz$9FC3GLNL27H_`0Nv8_cnW* zeZS2EZ=<{Sbl+3Wd9lx?sLQ9|+qq^ku{^%LNq#+VQU+z*@V0es7w6Xt^TAvD`89YW z=hVs`s{VR`c_hvJz8W|SjGR^PEBg7_p{B=I$I8RH`FLXnNwZsO$)7qN&YN-BzBeXG#VJnT^8k^GEPvzE)72-%#{UlxM+jBN=MRuOLmY%o3P7H59IPI60*t{(V zZ*L6jQ$O!%;C{V$Zo0hX`(%;2L~}O0Jv;b&Z`i!80B?`3tihY_?h@_AizQ7D#qh@I zNm|pbucsM)Isi#ax@*JRzWW|^smtkoOSiMN!g^X7%%C!p3_(M{d)zsy~@0uR^hGpt&w?S58O55 z^`Q0Fo2Fatlv=aa*W;UC&zo;2mf!Gp%}@S*pTpa~E4=M|*@HLVom%}w{}wDHe_YzC>>UMO!zy?go~hqu!!ysaP6FZ?~_?l>`Odg?(J!(09Al+-2a%G_@!9yIS~ z4sT~vcw2HuWZn|(CHmdCd-KLmI!Vj7Yi8s5T5-eM$ou|sr^DNs72Za_mXJ5ApXkhR z@4hO_ijt)#hD@-VLPXY(dy(`2Tnqz!Mq?tAKNhqtpUytP_WgEuQ@RepHZ z8h?=WnrcXQYT8FoDD!greGXif`-KtDB zmPa~S{r#4rbE&HowF>3!*mDox+u`lp3U8~%4aj~^Q%s;9a%S_Ud1XR&vD@;tX4@-A zIJ})#;ce=D|Ixe^dF~;6hO900etXA0)Boo1c7BDozAXptx!pw{2@8xGA4Cc5Ak%oS@So%egCLye(3Oa zVTHGi_gCS~%2QY0|HnGPY~FUK3FvwA-8KCsZFu|akb6fsyj@h`ZBd&+#k{BNEfzCU z9@gS_{?ysLNtu~}TJkqmoW1FJW4Fw}I5p>7iW}bc9di!;de*K#A98VpxA9X7;*EXQ zgU-iWJ#_BdLq6VotCakPw>A^cUFFi&*mg0;`6%B8*UYoGa;qB52Z(F8U;f?*UbHV^i0|P|KLO?Wn)~hh3!a(h@ODLox52YKc+2=+vElptdSFS@gMQkYY@7_W zX065}FzhHqT%h7FZ}lZ9Nw<1@YZE>4c?S&*DY^S|K(GVzA7<5zoezLQk;1t zPe?o+Z&p`^x9%gSf574GstRvgx(s#SQ*z6kjHjLw>&B|Z{>!Jp2rEk3*RRRLoYc}< z#_6Y~N6#BGXf&&owBhZQ#q)pX@OE{Dw@2qx;myjE^zMqql5d{!DfsTxw~BB66n;H# zfz=If*ZlWz{B`GD|9|0{3U4c)h`?Kp&q4#kA#0`i@E;SUX03d@NfkA5R;Ql~(?d^YGyI0PPD_9FYn!)g!CR+i67rVOTf@FWK5Lq_lBYa>O^2Melue3DYw7px zfjOzAJS5Hil(7>@vx=lkYYlH5e%rR6&09csGN+$GCa+Pt*_Zx1bwz#BgoQqS~IOMb@br=~~J^i#XM z@vGEQ<|5Xy&0`inC{W zsF}y*jbD{Cv$3MaFL^@Z?17p&^YO+E)KZm_=PlkoJqCZ_x%}&eHsEbe{}GM*o^mg# zHD{HNw-g<+E9RHt^k{Tk-ZCm0-ro27>kqN-x9h;$(B%nv<86@9XJNl~U=R6tlXet~ z4R**{)RKpB>U_MhA~iEpmpq2IYd&x@{(^n^erpTfy1kT;H}(^vQ=Dc#-lR^XICrh_ z(67;&ocVauRJS~aw*#jx*kRvq?ZDgCK_kQ8Q_Zfe-Wu+-ka$Sle7uEp8*8oat_$Vu z%B`p4AAl?0Z`Xsjb+<&|jrT1zCoIM3q4xE2twy`Nu_Co5TM7@klKPS2>?ba7tY~R- zzYUxD@>ctPYY*O5ZI8ekcPD91Px8cyYigBGLF3eTVs)M(Ph8$IIyAf;_4ntGvw7

    >A}9H$mTDElTd5PnTgx-Y9%J*?5xjN3tsvg4yEh7l z%*knM;^Dlp3UxF5tPQCv`sqobi{Y)bBg0#>#S8bfdFuq;TE0?+H{WT{Pc2PP@y)<%EG4Cm- zAx%sZ*W}T7!tXatx2Zk)oszmlT^Zh14Qe*g=B*2OyYJ3|c$4t}UMe zBT_csICD}B<=DhW#HT-53A5$iW}Z;oW9=+HgDa*TjzT`cw?=mt{eA$ zvra1M4e#=n=hdWt!fe#BJW@_gerYY^l6HBMDoXQb zX0Elgx!(qU>gDD(Z@s|V18b}B#{97~_Yx~7^Z4f1-*2qaSlV}Y?9=6qUyY?p9U9)= zyXv{CY~Ff36n8)^d4^Rb9i9rj{m;#^ds)sh-1Qc-!yL6Arg| z>kHnxJs6oc%>;7nB|7KXyw%$-Li#C`x8FX}YqHH-Kk&BWx^da>sfG>ZecQkc{nVOT z(&QPZmU7k)Z<+~kNAy!?@08R@&cL|2-=-dQ?g=(;{lVMDsS$V!?6ZMGO&(5OKE=%;@E8=JQQ;BDE48oconq|rPP`WdIzc<7I%vnS84sAE-6k%xU! z=i`kTs53e@yj}ag`^VY54Fqqq+m8=_PcwR`dUwUTqkQYqpUoSclCFncNWC$hk2hve zYQx)gtIvJA&D$XGHh5+u@TTm)Dyy;9hVvGylhk>#d5id|F}(fik_R{2ybT6#Z61lh zo7A%uPd#Bx{(QXE^Nyrl7s}hO-+A0YHg7|~+moFpmCh?-pEd8I6t!I5LUyWItMPEBzJ6(Kp}cijKjXbN zZzI6l_PI59<1Th5?ebO+yCY-NA-(B&3#@4Fw;ijyo^10r61?62WJ2Bo`%ilB)hBNu zTuIZ`^X6Nf6-(Oic1Zs|Q*GXE0B`epPRf2yeR~M;v;UAhnzb%(zSZfcW_2k}Pih^` z8+So%b#8e3!oB;Cw|N@{-Ucu9;4Pz8^t&jemS(NX8>gqqK!0Ot_DRit!g*u13~hKj z?3!Vh+PsYhZ_S^r!JD*;jJna;lWK+Zmd%?~u_?w~-a_UkO`w$1@OI_jkL_UdHU_*s z(r0r0yr^>lUTjEyfxOREOp)N`z>O1!`o{|?={-yZ9I7E zG2q7kBtuSf_hU?&Mxng@Ys|)P+q_KzZz~2*Xx>F z{!>sB`)vw%+dgb+G4CmR^PO18TB%m!@|I#p z**hh5l9OFg(=WwK9>d$`Kl?%F=e11*Z)5RQZzdf>fE3y+#!?&)+Ptvk? z%}#P?!`nISu5$i8)oI}E=2vR)rrA-B-lFs4+e6aHN8eq@Dd>3%sZv^Ncsus|w?AjU zr_;gP;2Wmpdrw1p^JOE2ziIKt3HVN4&l{`c(1y1eb51zj=4}ReYkgNv-g5Ng^{Mxr zs-_?2%*PuuXe!o&r%>KL`}HpUZQf>rx2ImOQ{GaaR6gE9x@s&|vw8FFD#LH?w-5jM zPfKjxZUS$&kC~qDJq?+BJ*?fsTl5JS-lk0d_(q$zS>SE@Js!Meu-))h@^9&qE5 zxR#oE7}sdlEl*p@CdHXSs>rp}^fOMa$*;*nPxk#rr>|z`C2e?H(CM7VZQf>sw~;Mo zWZ!R^31~dri?5$+W9jIh?uJiXtPW%4NynSjq2X=cx%Yp~=Iv(i)^0)#-mIKer&rZ@gD|rWzfxR+C@yxYK4Q)KZ4jap_Ym#if4S z`z_J#3~yg<@W&8z9(|E@YA=YIP7k~_S2K@B>v_{;%jnkd_RmSreZl5!8F-u6d3N0QRMT@k zcwFAv?k(-lR`)My-ZBEo7}Ge=JX2-s*YRh4MCJ z@vNWOysZFlJ0GjUTgaJ)Og?0-Z_h4oMOOFiHpCy!n{qPdep~h9N1dN9vl6^L)a~Z% z_f%P{;eHF5jPKfz-dx_K>LFF6oJGb9g)8#if{)S<7RdAOFEo)8_5o3i>_zdSwd^dl>{?N#ROv&Yemg-?+q>Z!P+%^}KNvwbY@tQ^VUeM>qSM!`r_rymjd{C;K_I zau&vhv_~oK+h;!Bq$;txlJbYd^}Ge{!tl1~zVq?pc6Z%xr&oB}y08Xs{dE-~Pikcp&B0bd1W0Il9O>(lr;U4=32?axa8MN(&bIc z#)^_=Me35@@V2erYZILJ^vnuxoBPhK&-ax3wf0q𝔙XA8(q>zMd4jV-GHGQbj4w zY#G|{HfKu5bq;T5Rd`#uvD$zMOb`R>AZ@-A<_6_tL&+gF|#`zwdH zvn#w!e4z?&%KTB7@tvOJm*Uc1!g*^Z{6SHvBICZ@a;>ip=S|tI;q93#uE0O%uFic5Jy;!-V_H_0hgm;8*Ew7K72d-v5pb?&!|D!erxv7r9n(~uiOnxV^^vz4U%__XD zdbJ8~yeFky_{KGpVa^miFlRPzbh4||T2oi!;k4uOmQmf@Z=XE=u`bU2c1eY|nWGlw zdrw)HI`xV8=4Wrz*}VByk^HPEc^Idk+SfyWKHhRvH@yA(^D|y_c)PU1+qk=P^2QzU z)5Lf^>+&hEbL!Nc$Eu~tlaIF?6EM8pa$xtLJG@<1;jQDF5qQhdTg_jSY9^`ius8Z$ z-smh+vn$`anzhC7R_es?Hgm|2pKy4)yu#bFV;6t&SouTp z=y_x3)XM6Hw-3Ge$Wn*5D=NG_cyCVLls#0v$26bHko=mpae0#}N;{R}%%kzpFKIn* zQU)n*b!d2d;h{fX?C^GFg}3?FF3xwqNj)@`crm`wQYN#mR=`%)Z}3GV?TJ?_-lSE&Rir$cxHJLHT6$dGxC?5o%ApN!=X~SH*TCDxxBh;8bNqV_ zYj4a6oYassCMnK58V~)F*5t_toK&%~;@l}U`*DHeSEW_-P-plJaPvNS+#f2ym3eCc z-fns*GH>jyNS!f2*@CVAv?`bRW z_SD82yzv`KOJ*&pf*!7orJ0{v$}f4+@y3eO?2vJ4dSYpp zH;a>+?+81jwmLMtP5#C$TW#L12X8H(sKFcaQ(Lua_Dp}QJk04k!}2MFbfQ@+d89aZ zq|wYHX_q%|Ilb;N_Nmdnb$$Kpp%~s+u}ICy#N{nx7lyaL z?*Fmz_Wjloye)dV3U3)XbMwXtq{#1^vwR96UHPs}?UyD{C~uQ*S@{i{w@%<~MDLZ^ z@2O@=8jm!MkhM}yDb8B@`;Dul%#8c`xi*$&55@3Ss%Y-FgZqC8f1tel|Fb%Sx0Z{m z@FsnaB*kM*DRmzA6k_d2%ENf9-TCGW=S^AB@OI3~NptP{tqXYD_FNU-LMFpAmGWpj z(%M*Y{r%=!G4<{=`RU2#P03Q#@1t0yG$q4Z-z$#4)8?%!c)PFPE!pp>@3fnWKOJu& z6R=LO>M3jTGfy^e%*IuWQ)hS#Z{J#Xg!AW-x`DUZ%OddR`x&8Mqq$#dX>G<#8cxvV zP0A+4*$K6;pKGa^hjESOT0L)ErKw_h3~z7Ve2VkuExUua$uCCW&8pjQri^PCpg*-P zpF--cQs>u9Nza=mo8&3&)bKWCZ14ZF-_su8t^1%=#k{BWFnNA{%csDIG|5&0A0K_S(uCylHkVd8Bx(6AZ~yK827B^{`em;c(uRJ2kwWHgniY zo3~!zZNs)2yeVr{eM9ScZ85xQc<`N)Z%&st-wetQ4R1eaJM&hXx8C4w@zB-od#b!Y z8yKUXnte)fdNevD57(AYfe}`h;`C^=l!tL@=9J>}Xtc|lCNn+MtjKtYVZ+<}vCCey zdFun-X5U(cH|Er6)|Ip-kL1zh)OeUbn>RYCH8%r0;aXp<$-{oAxz^>)w-cq`@U~xv zdv39L>kHn-?2N#hvY&?gupa#7Q;5|G&pIS$tUTq8{YnXL7TodZ~eeqr;)e1 z?jXB~1nad``wQphe?SIU!*Hz^w{TH5fo;0NdQvU%$d-d?<;3G-&{*X#bw$D4Et zQaq%?SZnj~CRNYasp0LKTmE*h&D#L*cHe6ryz#pNsUOM1xUXg{uGPe)`MA7EnWZZ9 zOIljXxJFC)8D}2GUEcUrX%*v=w(=O>zW1g79d7eB5WKA!eOtQs)as||{iE?n`(>PY zB*y4*d1E$eu3|hyvqR=*oZ7b*bGp3wX47<1@)+LkfAY8Z0ng<>k2DCp%~_L>H{X5v z`ZaTqa%S@ul3A*#iA!~*xF%;fZ<;D4kKyh5HlMl6zTXCew;|1M55M0+rlH(R)w?ld zZ8mQqnd@zBT;4Ly+VIxzl;-!_ybS?wZN^97E%k)CM{3`?TpO1+X7=sM*UwtiQl7ZH zNmWX5!`qpI7S6VL8w%c@xUULteEp^TsXdzfzP&N0%NxH+%_>ry9%}j}O+PjLj7wVb z>v@whFivfCVtD)b>Yx71=4}{wd${Et+3%^8pYf~vT-@;9F>dwK;7Ji@U23UTHgD1@ zDXys~tqqB@uB2Vw==815wbUgB3~!$~`Sb}kZ^Oac{gZ0&rs;=usWoe3d3@_;^X8k4 zyU?tqhnjhOwUo!@O_Q0~sJWK$k~X~E{iijnY~DtIx7#1^;LXZk^S+{A>XUJ5X|3du z{8C)c8#8D$JM`7e&$WzG(?d;<%NxHctzw*7T3hlM-fr9UqmedmBf;D1>+Z~cPgzUS zn(tbrwZ3spEse+J&9{@r@@sa--dx`JRgG3wG`xLvuK_!4-fjSIlcv_-OcccZI*FMNT{8`v))nUEX44_ML`Qm!~Xw80YU2P`kWwmDDY( zOCBqa;q70?esh$~+Zgcnp+&_|sacU`#6xXbE?>9|0O=ii%xTM*G zuZL@?UEcUrYI4Z9MwhxZyq$9Tl&LmvZRWL(P+-pPGKgsp*k4{nV1*VbZjH_a+$pr3K-lIJbnzI&MS`@J`Uw;et2aopXA(yw^X{X$ao;R+_p$%`p zeEUl0-?y9s-qtUy!W+*eN8PG-pp>VvYQ^QP$X#eA$$kpu?fJ95<^200Q^DK3XRGig z?TB&iNE4?gL`!vjbH?S3)nj$YoFO`!w}{=o_3MT8C;i+0{%sm~8`}5Y;@{JVdu+H% zH96@?$6E*+rQcO(*pY7@?v87-d80E>8{W2De)Xj`Z_~lswM(n;CUwq!G#>h?`F*C8 zM~X8KHFGj9>GCOPR;BKQ9ftHHtxd-pE0(5YcpI|%p}*R^%>ZvtZK=W=ClI39kEBEL zOKYWCjOV-G$R4vvok($ME#uT7d0gJ4%u-ycZp966-~Hr;r`f#C1aIpH+~>ZhoT`=6 z>$VT+KV+?@=X|_rGKYAi$#WXitR=;>d1FOtW~R>Y7~Wo+xa}i0Z#RLrASW(d1;l@730**BgN^_ ze1hl?(JpTx*)rA|-rndsXNt|+T<~_!@OAm#(~P=a>@`-NY~FmkDv}@H-dx@ib!&Kg z>4*ac*}TmIZ?kW&Q{EEIq~Ws=mp5MqiuAj@N!0^qZg|`I^|!xl^EMy64SBT+Z_DK-JkD0 zWnGPy_L4fz-dL-A3es2U)M#t+4Y#OfZHR~cQ!{5cZ_FlDDa8$MkLo114((Vjzd*8YC{Wfol!Q1k&4`jcm()}q-kMR_1eWxw;Q$7WD zNX=DEUAw$-r_>=kE#)!1-L_@(uWjCzfVY|V*5ECq2hCdUm)du&25QclC$(9eJGX`8pD;BDl!4`$zQQg2e6{ZPm9OL=_b*}VB? zt7pHSx4>N(-sb)60O!xYEdy^|Cr0MYw`YE)eAkBfn=Ws@6DyRrOP)V^nSH-42XEWg zH8O7@%!I6My1e=Bx=`M>-ZOEz&D#p__E77G-1pS?6T}IGtYx0mI-IwrwNp~3nQe~U znfLTBKY!mEo41wVZNZIIc+>Qw@#N^au-_>S=S?#KjVJY#NSc1vIl3~ubzb`0cT{-W zu+QEn|KXHVPnz6qRp*%}l+JeKLkV%?u=z^|ZiR9BQ)t#I4_@koPE9AwLqFqEPR6O3 zN79m?@qDqizGAoT|oOq^Ee^V(rd%s)h1) z^>42Epu^kg72Y1(n2@(v6J{PxThjDPx@qyook}~B;>GY*>d@S8SKmAQRfo4TD!i@e zup!@jD($Ywc=#z4*==Lh3g=BZXT#emN8Edq!`qn^-X`4C$h;}{((p#dI<=;CzlH2N zoHu24!`oNdO?{`s+gTOfdOX?)yzx6;jaKfrCX=icmp5kC=vX__AKjZLrZiTn|9^b>;e;jXt_n7p3i{b65zb?f8uHW_hz2{YUo8Ntt z{yk0fxsdwziu?Aqd*5j!%DvzC)ri{g_UHX>?d#lc=T~?eIll&P{GL$K?3o_MV`=86 z_Ra6|CS{Z2tVj>8G|lhPP3B-`UII?Scw#gPyLzoA15B9weP2 zr&LRdySz!xrXF{BOSB`y+rAIo_8W({3oE>}?ESF&p4I~onwdy_N^#~)$D5|Q#=~wk znmMUm-uP8&O%=&g%42xD_^M}bw>zGz!kaRG!}s%?9@k1bqvwX4 zl7~52SL3Ird}B)DU1++>@EhL#amTvh4sVxKcpJ8?3U5+f)}ogD zvEnIel}{m7hehV`okliq5j!`${b}2;++UwwTH&qriypj1?5*az$y)rhQqv=8$zMJN z>8qM-TuUvjWt=}=o?6Ny#hFvmdfq~+l&Txv7M=JRTyML+f4i*0+p7Z~b>CB|^W7=l zbniFcQ*e3ny&;tq4R5XAdB9l!uKef8FR$>nX{85m?mcsf!`l@V-X7TI!5jN&>N=aZ6zASp6LWb>)QRD( z)d`D^ba=b6!rQ{3oA5`vK;WJW`x@zT{zC@=NjZDTHKTW@=4NdL*sMsqwhH zX(~#Yxz_TS_jJh#$Kb~=?E3ZTH5K04k9;iMdn)zDK8w`s$K{P*)pW&Alg6*{NIBUX zwUm<{mp9ES$s@%}6%B8`Tmi1kTMO{E;k5|7u}{reYy8YZtyxQtr1iW>)uqgh6_@%fl(#ef z`u!tp-mV323r0V#e^7Z%Q%e(;Jic+}m)6pw(JpUNX2z*mg>h=hBgMH^qcwT*@y6<;CTaL~RZx4O#&~MtjwE}M= znm?hx-*WWe^*>6B%%9C$k>sb|waXiKM;+LS;celx2dtl8|A$jrgSU3$J$Uo&P4YME zb8!QQl1JLR#!nC51?p_xSe@Az*Jx=iJJICPc$kygd~(lwS`2$3+=i?zl(+4B9dw}0 zTU+pU)1(@_@n+_(eLuNU9x2XR)bvx+BWe1}r@#nTh3w9EE$gPx;k-#xVz(uac~AfK z&$~{yd20vWMm-xJ$UPR-BWSj(~xP0tmRBJx_k;nR@8JQc|!Wk$6JnyhPQWKw(0Bk{nj45 z?U)joH)&$N@f`d0`ZaI%{pQ;%X*`-) zh2L-NMAA|x^h=&t@j`jK{e$hh+4oyV@OI1fPwU^))OSv-2{Vt>bN2lvtzzcXnmtha z)(z)Ps&2*2{dVDXH_fzp>jd7$%<$k%vs0^whI>h~wn$GlZy}v<0@Sf``sNAejTJL? zWO%!1$tgW--a3P~_76wqjlHGN8T<8OThALSQm5EmNKUD@xV&X_Xn5<<>V>H`Z(YFK za~+@2zo!|sYM!<4cPzesO>aCAJ#U)Kl7|&j>`1E1xXW9LPD(iqZ!N|z`lHQTSMauO zRt?@bG44W&(?jj+=UR>S&0h>}n$GFrE~x48)h=&RhQNx3xBfr8^rtp&-N4(d%~g0K z|J3YL(;L@HnmMWIXFQfJp8~5$niVAv-+a5(>`3ysyd|n`c-#N5pKi17w;tfF=iJ?d zH%?nK&+;i~oXi}ei>$>yUEcWBh}!UW=$!}R2O5>{x1Qi_=M%djZ=9iY%8h%!vBT85 zP~N`%zQg<2_ggRUwz0=^_4%HX6Y5x}vxm3n6EOGNC%!Y}^EPk2!Q0XWb;?@^R~kOE zr>)5l(qWNn3%%by`o*^GZQlBTx3SOchP-J`Gn+R}hNj}t^A`9iGxys!7tFlR=B+Py z>)Pk}`g~7QpJq63fu~dTcgRC>N^AAJ1$(oRc?ccY!@uJq1lp z-+sb*({z=>lW(V_PU?!8sU^P^H@qD@^X^~S_gjDPw)XiNyyd8yn;EPBqa!T<+c{}rruYTCR z-v)uV&M!saE!HH9%u@_+MeazdRVZ&4ZtQl4&D&t`wtdhGao7oK7o5MJH3YofdrKqp=DROWQqr27+4q|!1FQS`xmKe? z^5^4C)0O2hyp8z3BmZFEZ$rV`P1_Ul#_wRM*{2kzhnjwBdNjIx3Q`6sE@h@)(weoB zCnWCjCUwYoj*5o2nV(zpe>QK!z}v)OFYey&sWi9j`%PMVmp9)GO26Ul zjWs8nX7e^2y!E`TiSZWeH1)g%e(Jq`H7)Hf>1w-Mm&)tyy%^WB}~ z=SfjZQKBf;C_BVWpXPdU|) z=}CE{IBQ8BDbBT$&c|CwMQN=RXXo@#(6r6=igHu1>Q!zUW2!c{0+=BMNc?yDLyGuPR6M< zlP{FFfBg8y?e_gP8oXUUW^49)s@YLf^knnqd#3d~fkJuP_Rmj`w|N@_-nOi%!Q1XM zc~X(h8=WD#9;R0)Z~OFl$1t0>vEc2V=G(&Ww-i%Nk(1=;c}v|1yGo(6d5gF^^Znb1 z+fRMN=4~8!yJ>s`-cn3SlQU#idfuc?LaIwSL*i0D*}R2RwALEl_8RxV2Aj9>;BDaA zguF=;V4wUw9LA|NdFZE>@@MlVtzu?h?Yj%f&$y<~LU}vpf!}nud7A*EYpH$ruJOCPrLL&yDusunxx0>7(eSo%=eIW6yiEpgldjv* zxbJCWopRF2*VXTYr{j%hSJH;JvtK`Wn$6pd;H~@AD!egI2>+VzW<&C8)=E7zUEX|8 zCmnCr{0(n+z2iO3&)b~>-d3FlMoBQpH=Y6@i&D%8aws1xx^JeuN>*WN@HpEla4oQcZRo7CvJ0o{^@k^HvVA`-eR#4E048bFIK~OWA%F25&NMo zl(+YNcg*|k`)vkz>)L6j`<{kOPg*O*Ihhn%s#`t<>8tES*bKcC<^k_8KQftdvJ3N-k6`eVBA-0^2G8;y%o-lR^rikYQ2Jt11tkK|$8<&B+F`*z!`&&A}y z@OIZhEu5dfFdMuLog0}q-=68`-l^%)Xii)5yS&lK>eRj-`a`tTgC?iTn`TuGkKwJ| znTJiZ@3))5TjwV|c+;F?jz0MrDNJYc)>L+(>9bJYUhdh}`TlJVc-!9N)$sSUDRP*P zH|~_0=Pt$Rk#xvfDQBU)UG&aReB8d@=7P6%3v%+7VlvXieB*q5sF^1pZ_FTNriYq- zYI;)Wbi7%+Fz@NR51Bp6=4~E$Tl!2+-cn4a$eiK471^!y>6YT`Egf&HQqqRE^S{%_ z`Tq;^!P~e#uetB3W*1hihVQ$iwNhNllRDnGyoF4|cdec`-^|t-8QyMrCOb1HS%?OKjd2fwz16zwW-LQZ0V6sFnQJo#ESu1%Mr)K_qyh+)Z+0y2I+j06W9qs#V1$dh~?9KXlPkr|&`K37LlaDuM zre@}P(|X=~cVzhuZyVk|u7l0nO7J%Dwt{%GdTV%bVx5j}{(QW}>Na&APD9U|Z&$2R z(uTKtkN(WJD!gsjXYZ5$c*?0KP42d;^UM=UXa45S8o2SdSS0P+rX~-!;+tRcvu--v zSViij$heD}X0{rSwNr!JX&>r|AGf<#nYYs_ygf6r`N#2xG{aB9>Y?g2rtwH8!nM@& zYi1R`Q&yyASJdjYjopUy7AsFk57OG*n>X&d7~V=1 z4R4>EH1HsYw=*id4IR_MeNTDcvUA3%>7mwmB){)k=8-sOPCaj2#VS%fmWTPNnP1ZM zyS(wMzM2)ewiGwK{qTDqeA(gc%nEP)@2SEYbMCHM&s&ibkUl9PJ-EDuWXo7Z_-#BwpOL;UNJ#U&StVr#kI$NBr>oqaEJPtME2uauwbJ z>(=D39`fsXtA}pMp>IDfZz0(-svF+M{7>_DIlP@;;jPbuIeClKGe3!vmi9=$#zTKL zZ*(%V6sIRd$C@g8aCu9#3&Y!2Mn7}7!`lTF-mYug%6(7StwyJqlBRBmC!4pB%=Na` z<&7Qk?5HIVmLy*=Azzo#KnO0hO$zg1r=$?x)( zqT71N8JD+=9U0z^Yj^7l4sX{~c$>2@CvTeXmoy%!f3BsL{1R7^zkCY5tE6n24yCo4 zcu1cvZ<=m39(G>x8{Rs8{PXy6ZF_Be>+jb$$G_(=`q`Yog^YlWYP6=7#=}~BF1Y9u zFu=WX(hc|n+O9?74Nw~&hLMAE+bnTKns zUEcUrW{@=f)Ry1y)^%?0x7ob425*a3c<^TBkIo)noYb}Sy!m$J>n~=fq>dFMc58T> zboEgDg_-jAZ*9QasI66a)AXS6a9{PNvw8E~ov%OjG}sSG%igu7D`vL*hPRL1b18m9 zq|DoO;H}+|_WJkK%HMFmW%SVC;aXom*HZiDp}#zNMtn0%Yndmt*7T5$H>^~c z%1tf**XFGQczgMk|7hN)d9r5(jn{D1Yg0}~6=otQ< z=IG7qWSKu?^30R(ev4I|out-rdCQo9;q5bkQt8z0t_LW%OT@w~)2CjK`|j_p+Zdm3xPA$jug7E;l7E&bHKBxLjEn>oX8csstu14r7tbp>ya z-{ZksMy=eRKHurZ^5^3%R`nwDWb@|R2^pX+w5#F8+QC*(29l z`}JbJ{&@@89jBC!w;a_CZ;$N%=I?FZx`Vfg6B6>K=~?69^g=XiH7;-LBxFa@TIP3o z<5#5&IpT)5X3zc1`g!7iJf#PC>#{B(Z&J^R_S;17w~%RQ*1Eh&=MoqJHDU(?tJ6S zFRi5~g?4#MQKcSo=Ho5LxtROy`%fIv!RBoscCpZMaWD)=DQ8E9V~GqEEo^ zHsBrm&$f9R1m2o=?(V*)JZt`%mzo|)`}#F`LUM9#dD@If)lcpE>j3U9niB`t9k!hn=BMLc}kAv@Lle_JWP6qlxy z?|$RCOPQ^>;cdWSceS>88xG#OK2?P`=3!mmoiYzK^H8%E{Tl6?Cm(ODLapgA#1k@^ zLV3IKQ^(;SC@=qeH6y^=j-EaB?`g=y>TPX4-s)`^>__51A8*V|P4cNr9&^8)eC#(a zx9_)+;BDi=$h=9rX3zDY8<)4#Q)2$Oyh&$Qs%Ur{^Uc9+ZQgDGZ%dx7!W+4zmTIMr zr|2OrZ#-Y>)RWQVkIP%e?hJ3A8?xnNHgBWA+qk~Hig`~n`po^iSej`|o^amkZAaNV zC3TWhiYs?&czbs8NB6dQ8x7vNEcM_`S+`O6k!B^u!+Dc3OYz2<9!X2OYs1^%ai^bV z^EL*&ZQtU-n>5K-@mMFK$&+;QQ9f~+T}YGX+PJ(ivsK;j*7lLZrrNxX1#fEy^e*N- zwR($kA|W}$dCQo**X}fVG#+Uhdfq~+l&Txv7F}@g5jJn*z}uV^HF%5ly*l%7UyM^r z(_@@|YI-DHJ_U9nIT`o$bFD^8yO83VoGx$7#vL)P$;`DSzv1oHX9sSxc^ePj#%!&@ zo0YTSlk3}?)W4KRiZc&A)EZCr{U+^NlbN0n%^gWv%1^(`8&}auEqNHHF0D1ZUA$=S z?`+;CfVUw-`ur!qry8y})uyPkd5gFs!`s`=`@8e|Arrw{o7GK>w^)Q^-*1{zkUEj# zAvr_V7RuY*W8S&ezTYN+wcOag105N zSK-a~u3+7ewak-R7k|GMIT_Y+dE-~5Ip^>g-o9~RmnAlDQ^4DZSF7+Q_0ZJgoMHJC z81da*$Xef8A%2%PDO=!<3~xieIp6v9=~VD`-Kc)q@2S+A6qkCI;!>WFxTaR|_gl!0 zLe}be3#n3CYj|t>kGcPC-*3~v+tYW|;EnG%wWc>;zm$`CsOe{%njUIBZ_IXepNkuQ z2a=kZ>8GZLnt3EYv=1(L*Gei z{Dtzi{#U)7|GzK;yxlgozx$qQc2VRhNVP)x%;qg5v(zo)zWF7;6lbk$-ZU8`591|m z-qWvN_|Zl7{WcT4jk~uBZ|pWivmZ?!`l)@_YWyy5z7;i9>cNxETf`0xZ~H8H&G~ua zH-WcK*A8&+w}?G7e793?IbGiBt;1OT6w2G3C+#=VzTaknx9t-=c;k1W(lqGNXem$X zc>VCkDTUmz%tKu$ZxfCuT@BvYIkkodjh`NBO@4Z~mYN>w@+r_MX|B>} zO`Z@B*OE`h*{91Jozz^ViPKZk<~==l_`V;pdAk|BJ=%KUp7+$f>v)blg;+XPeoY>i zw^+Nd@)+Kxee@TP*u2dFZ!2!x!`m6f^Tw0XFhjquc6m!w(eU=Z9*4bT^EMZ}O?W6L zZy`O0tmU*r@{~^@Bm+CCH_d&yyd~<^@OI+cCZ1{YHV?dYZZ~Mpd+OeAiB6 z8_5jZwc%~`(|woQyv+x1+ox6GjXh8Y_EU4(QY|UYek6|+-~AZlPN`kq_*Kp@P#fM3 zeE6%*zZbm#ysg<-g}1=EHMwf6{G9Xd#~Y{W?v&Kg8BrVFre1mCAMN{XA$XhBVQ}NV zr<_d0z1L(uE^jGz5i2K2E9+R9tvrUe&vbu}^Xt<^;BCN7Rd}=V)I7TsoW$xu%9G8T zv?|3;Q|HX)jXR~z(UswC$1P(Hv+uXX;O(_Xs_@2IQV&v`YwJNveP;7k51q&AA)B{Y zRZ4jbZ?}JM-1#R!HZ=cz`u?jOwaeC@aySzyq2JX)A zHs|OMUu*NW6ujMba~0mCethHXU%F@Lp_cqo57Ju3xi%kf%;2lp3AL0*ngG{QOL<)0 zn3FYjF^8&kvw4eEg`F3v*}uyhJE4>D z9NO^qvtJ+lQJc5r;B7$nq3(N{qgK^3Ei#Yq+_HJA=gxh5%jV5DTZZ58Hek(TeQn-W zfVUR&n;37tTuJjOhPPPVO1sd+vv~{Y(E3-_-+E6E`qmpZZ!5vuqfdG87Wlq5`W?vA ziOU-&L(Ph@w3MH5YL_>Dm0dATO;3h4ytR1v=RdCSwqc*WPyW*>r=B#q+p5kpPbi)L znqI>i_c;yucx7z8Cfadv3+Y5!8*4HyZc@d-xWVoD&wsQ<1-HG*yq#9zZT6xnylHlt z1OJ)~`0h8Phw>?CcIw+zJ@`2pmp9*C)6dM-P7QCzcRLq<-FX*p|E};h^tmd$S$VuT z$>^sc4?U6&=|Qu$dkK|JeJnp8BO5Bo2MH&$#ct>?|RLu=-SxATr#f0V=9nHAn1f5C${^6c9i z*GgKGlWR48dbqZ6d1EK+&{s1LPoMF0yjk5E-u`>PwcYXhMR)x3w299vwQ|4jk zj7yq+YRRvO`{tpanzgcd<0?rr8$FDt&{8cauIG&zsHI(49>ZJjGv~hFx!=yN@OJCU zD!efdHS0=oUl09~W-V&T&$#tlk`2FR$>xn2Sdm)tFrGrQ2WqL7%NtiwOBv{qbZM>O zZTzi2#6MTF>wY_@!rS6)Rd{3FjQkB}M-SJg)_g&vT2gN=Z<3R7YN?`cT$5AUsT5}q zE^qv5L~VF$w!Ycdo%`+F3U8B!j&$EsPJo)dG0t5xPAz$u(^oS;*Gh4ET;BLqjn-6Q z9?8!*HP?n{mp7@3Z=79~)*9YEcGRQz@u$1)xAQ8zjl8uAZ_E>-$yi7pu9ap&k4C$^ z#mX#Im!=U?SCiA_P3lmJOC4Hq!`q;RYrpH#{Q;Z&LsD5O;Y?-Ic^f>bl{) zDLXX0eet)wrZ~J^P~mO+h#TDZRLUPRvBp{}?WK6$q)t-D3*~M0$Ikq+!`p=w-tNDn z3U487q@G@^{7s9ukQ34K7E+~jF6Msg)8UW%JG@;~;cfnFRd~~!DSM;VtgQ!6IB!j5 z*Di0dI=Au|-UeRZ-2MMq7gu;2I(n4;JtZert%lzlp{Jg8IB(otJx{~sjh#o-hPQ{8 zZvKmNzcs7y)@Dr=-uUYdYE5tavj^0v*D}Aa^c1OmdvJLxvfEg- zG`r5`O_M>{t>NvNHb;HO;q9^tZ};AplQ(4#4d2(r+E=VR^~qbT-SKp?d85-;TiqJo z&NyVn4u`kPE4Xz{gZFt*xVSoH{HM{ZC#LDa*HfSOo9`|(em!rRY#C>5 zc>C9&^9DKh+m#jGdOZ+6@oe-d;O)-+m5nS66s@U`kHjQZNvbQ*#RR zWb+o%iSJr=<-1nnFO;`SHvQ{phqr4gyv<)dfrBR_a8P$JaxD zIB?9yU08AR4cxDuIec~Hf-CdZ0=#vZ;lUegHFaG-7aSQ*G1YM1*i}ZihPU?XHx0LW zyB54{-?WFfDnDO_6K?7{E^isrFudJ3uE*bP-dcjU^&Q7&zo!{INB^zKjJ-s!RX_Ke z@2-7Mq)^_@`P?5bvUzI--WJcQ!JF?inu)Eo z-dcmV8#eFZZP$C6W5UsI-4s3K<1NQ0!tl2Js2{&#^VSBub?7=F`#oh{PJ^1BSep5% zee+9ev+pZwX>q^P_5@g_~L7~V=} zWO#e^?eh+_@3*$#ZOs!odE@WBP)lrRJY4JR=UR=9%Nx&{TGNTI$9FQ}yeTUh-i}@U z>pnJb?ZDfD9uxKNsqbX!*&mlTX{RCVg{);i;k+?hhBmzY_fdCQKhNk-r(6%-MlRS* zc=J7-VtC6qotnSa@ZFJAt5DwVy7P`t+xJ_0@YepB-Gn!3+EToJcnjf5v$jy)-f`$h zKWOvT0laPLJ*k-YR5KZgjTG_v;Vs4Pm@_VKJV)wMMf0A1bHSFIZQeS9w>69Fh&QYM zh8IVQbE>{N9d9YRm3F786_+=ulTzI9cFp_S{IAVhC-64&`6|5e*CW)L-DULCzz;pl z$y(HspK)KEjyK;*mz3PZ*HgBE5+pzwV!{1XflcUcD4rArf z^ssyIrr#;4)2zzjF}&Tk_#c0;dFuk++Afd4TaMlu7>l)5VpWRo;jKIYX+Am5#qf61 zktaI5bp>zFzPOw4<~yG~yhWdY;q6;ryXRB({nib<-81OM@b{Ew9r5#0^$wKs`^KYB zyK1+bj7Cel(5#g_<(=}sl2eLHU0HF%+os+h#$T{6|NqnO;BERXk$IDPmf}{=QBF;& z#dv-4#tvg?l9qJWhPP)focu}qe(M3=25c{gw~${4rC>7Y!=~upQ)x%sT~pC|-csz^H>aPRLwF++*w+)|GO%IYs%E_KJo)ACR>UrZTjg}_BwcIH+JsQom)LiTG#;YvS}UGqvPloSJK?Lp<5Mu?jV( z!MLPL9U9&SKiOuk&0AmaHtNotyh(NIA+DKK-1|*av9TuO@|LJu!`pLx2OMwn)(^b3 ze?2m9{3KF0*0iO%E^moWq2aq zkhP?O+Bc`mn{S3#{*Ya0dT@D5RMGG@@Xf29w|N@?-qx)th_{eETA!n;H-={N^yqnG zb!ukQXlVkBOPcxF12sL=E^qv*CIh?TTIy0B!`m4j`T37+-UfoVMa`!dbH8c!#r{)k zJ#Q&G;R#a5s>PfxZ~SV+iiWpC*Prx#o3}yW?S}CM@fK@evGO!7Z`?(S(`SB{H-1&p zIl3~u{pi(aZnt?G4BpzW-NW0Sf3GIa1PpI)u3q~Mo3|n0?WLA8^1Y|N{7dht6zAuT z9!b+rE%h0Gze$;;IJ=Uxv^(acN6N!^zWa?Cl-lstamH%r&yx=YZ|f)JfN*!ybAvw90I-55-rCn$|rL~5)ZO7kxfPKFW18;L5h`^iFXGol%Vo7WAb1nVl zQ;@z&CpG=l^hlb1YRNCfrPG&kX7eUh$q_faEk5;qAF+8G4&EkSH#6UR>e~;eM6Jmq zO;U<$a^}0=G?|O^Z~~Hcd6S%hog3cz4Sa|5`@JK;TmPv!d1KF-KIy0SWlyTbob*WX zY~I+FuV!^>DGxo=?3`;gapuY9jZS7}T&WFj=Y08E{J^&IpYI(B-db+R$s2nJ>64RW zPK}nzmJ!ZE|nwZDHG1>Tm;NXT1Y|4I35y1e;L)#Z&e zGMf~qCqou#aSM_NjuW?AF|drzvest z@+q(@YE4C7k0!sbN8``tjTtyMYUNH1Z{z>9+4=uj~0_)=m0-7kl8W7}se1 z{iexQ&z@}FB6e{w^+>hvcbC2TE?llmbyG`jgzZ<{ahQOee2Sn?|x$jP3I-Q;ce8v#(&%9 zZ6bJk<%!6=`Sz*Vi^k(yH{Yk*w~D6o)Si62X}YqeVR)N%=)Tw3yiEdc5B0cN|DI}k zXeu5(Z=5Q1>Yc`_m5w)S0*1H$`P|uGwRxKi-WD#XQ{Ff*WrAu9z`)%=QQ&-sc+Z6EDzt5cA`#qITKm2}^GRHc>Sb0cVzFjMKYIy7U^pBi> z-*PH=Yrdq3@uu8+!_6a|ntt-JDoUNCj)(K6?9lM`w@ZKaoPEDd18iJ@w}y&k|t+1Z_<2n#0_r; z4Qh3!&D%}jZQG!E+3zWDcxp{Q^iy+6jB9ezDeQgf|uTv{8Kw~Vd~ zZ|x56^+lVvS>WxdTdMHJoYdS=EY1AX%rC`#J@n_}jh#?4Gc`Y1j7w`ZammB{)Oy~S zS<=jGX~Ww;XB_r5o448EZR7STyjgh~-k?}<=@eq+m-4W#o;R+NGRKOu7IjGeY~Dhu zSZfV$m+b$EFWJ1^4Bi$DoA17-%tIZrYiVuj{8C*#Z&C&+9&1O;LoL;n;@P}Os~DH6 zTXDnN!5x;BD}2Rd{2q-BqXKO*1j;q@uqd>>->tX3Nlqx2a#g`6o7SbHQ8d zR}%6TYpR+&8V_gX^5&a?*?e`1T2f9uZ&C*CF2iGZ`_}a%{-4d;Jn*)8+Jyq~Yy_3m0^eAoKc&E_p)b;H{cANtI9ZQd4vx1DRM@Fw-3xxJ;e^%gImf~G6(n%ei= zHMJxU>uNOf#N~~dEp2#v{nrPzuz6by-X3ed$i3gJJazjfhV-fDtw^pyb|kGWl(+Lv z-{kyz(M!PFs_{*LH))a~aXoLoRT_&=)}=0#xAm=m_AdK=TMFJLt&PANcbDRHQshs^ zo9_ft`wQi5^X#8rW%ITSymfE2IQu=7zFY8}QoZGg%bV}+==YtZuRohNsT0PjGpd{K z-#$5N^apI-mV>wJCfDFCBY&fRUl-C_T;4)D@m=eC+h_CUTRp>Xc)RU^$6MLFtpIP& zKH$L{Ij5F-phu&nJSpP&c;nvzq?UFkc|vyNo73fu-LexYPEUq5yiI#grxR`7R)V*u zu3M7*o@V6VbHC-DfbSh8oq{Hh%bS!<6VIqAbN#E^fZNR?gPo_SHjM?5N`rl)w^+Ng&cg7v;*x_}JG}k7!dvh5OZPmdaoMv@ zq2Wc4;;~L&%9D?`9Nik;zViL6|J&j1^a^jU&8Wc}?@~!iJ7rv2%edrcT++-#E&0o* zKqob)%{a43aeAofrIb(C!_K61T}tNPrCapb?5nZO6oKja_qwJcKDm4E_Qf3yTaRo$Exs_qgGXx_-Rd% zlXGK!NzH&&tMG#J-tX{{zs(!zJmikx$47lya}I(~Da!`nF(-o|uW9{!$cCZ_SQ zf9^}OR^ws*?8&n_S81wfJj_o&Ye^pZHCoRbD^hE!lstyF%chRr=WDYx zzN+4(zH762^UYSzem!r2yD+@9dFJi-=kRy^e(!k|-gZ8@hqp6!Z}tl1?XR0Rz3kj? z=T~^!)N@7syr&_bFW&#uQqIO@Ow%Df)Y6WmcsOrTb;e8D+;1~}^z$Pf-Y%%{cIUz- z#v8xG;e@3)J=BfM8$0psoNF~YoHtFClE?7YujQt74sRD$c$@xg6XQ*}_nP11)emp6 zb|mE~l(!3SKkFcew~H#g4eYzJ`1dsRCt8|3<1TMSc2#7paNd;Nn)~ftKO2jGj%(Mi z7cQ>w)?#TR@WykeR_?3j-Nl@ec6pPWO+D`N<~wJl-|%+*QS+a6?zd(Y-X7ahg*WD> z)=Zw?Lu&k*{LD$M$uD`zrw}U}t8homPffq1nTMKrsHHqEZ<5nDZdEtD{nr~0KIrgv zNrksN2HX<&J>_J4`!CY}pM1Y1nzP~U!o7Fk$9?R2PcN+t(c^Xb!g zQcu$5O{y5!t>Nv?`__(kc)Ps9Tc;tb;=ZTSUg|9#&YNaOfhXv7V@v($d6P0I8~B$uBz}hYxwHy_mtlsQcGv8@knbman48b zYdm`1QtycQsiocd#h(MgJ#W6#uy$>D`^!)E|DMg;wcu^Z8&!DYgH3JaZ(znZ9YK zA^o_#vBQwe>`>CoPt6{rwTyEuH9gt9aTPT)Xmn|<;qB+kF2f%vFMt2m8oYJB_V#r5 zo2Jjk@{~^@b#-5UeEZSl&*n{2QB$Rq$MAN-_PZC__gfqAwsk@k-jw-szlq!0zrsWlTTl(&o59ccZ$wm+ZJ9=z>rduQ?Qsb)$Vk2HBH&YbCZlYULX>P6~8dAoI= zeLC3hX$SE3;Iy2)@q0^Z-%0YP2~bOUSc_WnOYwBPX=vAYq|SZgE^pG@IDcwoMf3ez z%QhS5+q`uIZ?|vE$(z(ek&~2qU|i})&zqEiRq9#0yd~O^;qAyHw@$Hn>jd7WbhsnUQAt7Z-laI_6Zbu3 zW=-`FKi6t>HgB44HJ(x@hPRPp7i_Y5>jU2IT;#!&V#oUQ=B+d>O|Gal z`StgkZ)Q#PB0Vl|QuV-{8s47%ddL3u{WcK1Z5Vjpf6jZ#dFpwSPA5g29mdM9=Pj_J z;q7BbopGVf+aU0^bY()`Vofhr9?sL{ja8_*N{Z7{Z#tYeY1dZV@OIlj?l{2aZ7_Hn zxy^$&>%Fe}J-Otudh=QsOe^!Fyhd1I9fZFqa< zOWnHKybS?wU5BpCeowhm>WsQoSxW8q?ak#ab%(L?=i4c%3+W`Iy5X(&^b7C<0?Yq? z$WZXMb#)EiLVBy`wM~~d->H(cvd&

    RqAq8{XdWk3XJo-*3af+x;&$0p67RtvV~| zbeb-2zSD4dlV%baH@v<1kLHKiybT9$^GB@9eoq5?@M3_qe5cB_8r`&b3zW@2FGWG|Yr}!g&i}Af$8OwT1H5VfIOHw|N^0-rB#mhqp%f^GIYS7PnHK zLU~*NhZCH?4|oH3dtuc5#k{A|G&uPpb@=_pZtHD#>@6;D(oRcp^ZncM{tq5z-*2P9 z+dX#|#2Y7&LQ8XyYGw20`}<>372h~_LG7D|{Zap0|a%Lm>2P4lVOczmbF`#RXxPBdE-t)y4`iZwHZBQrp?zDe^=8dZY zE1LUl)AD1Ewt1Tj-ny@^!W(c>BVI_~Vwle%|hl;O&)m>+`** zA)keiwVa724?WcN!yCJz=1zSz^Tg$inXRr2Z>Mkj>0JAMn*!d}P0z`jRjc9qv8EEQ zAKpSHmT~$GClbz^rb=lShPUYt+~fTHg{k0e-litTn`ZBlhw(kUMW2A-?Tk(P9An>a z)4Ch=bKhz6HXXdRomGW5>DS0o zoOLDbdpfK|&HPea<0+qll!0+-?t*bv)MzOWb5b*>Z`|dL8K`}$SZfV$?VowEh0WUx z@YZ5;72a4kg=U}9l@dsrAN~Fc$2ba#0_sJb~xdyHg7k9 zx4Y)l5pU9oWbCowS7E-Me7vzkYTxP_e@K3=b$R1gsWn|?cnoiCUiq(wZQf>qx2aD= z;Ei{zrhod`KjWG_TuZH4TRsI=WQG)R=A>p_YK=$ZcX{Kg5N&n%*8O(bd7rj%R*EwxHU0VSH)d-pIvsD;1PpHrpL_m?Hg7kBx7G_hc(d~7 ze#;sw&mP{Ae##7QhkXAlXW6{X0dG$~T}Ql;2VW+A*M?8tx1#TkV)^6VZ_=(yal_kX zt>1Hn&D&h?c5Cm4_3x=P0qY&Ft9wuCEu1%L8jTg_PO0Pa#?01E4R2qVIMw<0-{$>) z_Rc%*?xM>7h^UK&MNzzph~i?w?&^xJx;EU^6_s5@1%aOnp#+fLA&`V51V|^OlS&Hd z3F*BzdI>dzl2AhL5CZ(}c=LMh`!#dV_n!IA%=g|W<{w_4*Lgm3&Y3gsJM%mVNx<9O zWf6F@>NXW`Qg4-0kUp)_s@53K<}Km`3~v(_cgO!AQu+VM7lF57&(+~AVm}SxQ0jlj zyWgbGLN;&GsvL2{+l7n&>DTsqx){859`vaGJ>^|WE%hdOeB%-`QeDQST5)-ksxZ!7 zP_tWVu4UX;7s}g#N1SNydC=SFLbqe zTMFLpd9em>nokxzlI96Yo{*DbPS(=oba~UP(s-n<7^h}VX%|wQYo(knZ<15WM!%%X z9h&#F)7S3*oz2@a@V55aP1)}$cS_CPq&Ph(bV%K7-a<0_?lg5Rmp9)Im43t9J@;I5 zk9o|?g^<-EpwRU;ik#@)F>3Iv8KzXgk>c#!SLfp`M<<52nVru+%I0kqcw2u{1m5^Pq-Gj9_R_*IYf<~=)bqy7lGg0H zQJ#Ffv0KiWaZ4NCp8w{Z``Em#25$@Ah`^gww}p*nj_Y}A)Go+&NDtY(#i~;7(D2sv zrjwojKV%Jf8*$xZ;qNK=r{9oVPET1I-d6tdyZHO?H}1CUv43uJ+%Z#nuj?`Ei1PVQzAXZ7R@oM; zij4byOkLc3GlcApwInU&r(g0guE&k5a%h8F&j(iiZ}mCdrNZ0sRo;5NRfjj<-R0=B zh0h{uxxBISB0FJSU#;o0P~Ohif7v|_ZzoiFdwJ~R;m@h>l$ybx&08~c7^{bD-uUna zRy6lpm#J@k*5U2MDsLO^jKG`bQ^vi}&;3$sJW@`^nUnEw-lWW|B5BQ9%>*=gvUy{N ztVm5yhBmyN^2Txa<+ttcY1=Aqt1o?`nENe9|1Ixq#vXHf!g)*4l_sa}B=x*$s%Ll% zZx^i{bhvZBomAy*;)EK!`R*vAhul1*&RIDHX{S=0U1>b@H%h1D&6VzJt11txs+e(M-xxSo7Ju1ZNY9MH#)qXQsr%1 z$0zmgX~@K^-fD7^!Xx!uIR)Q}Qa0w%c<9&Yker3`_G+gOb#Zw6Ta~v5CwuUwnM^EC z$b>aLg!7iVL(MKUo7&+=bz7@8OFcciz)8tGwNR#Z&HkN`|R@`{7z@O&-Y~ zEAH~f%#xNm*ZAp?^7zK3oQ&&v<0^I{#px+)!`uEJdvUPC+gVlK?wnbJH-0*-oVkA; zL&_<|W7U=NxV%YD#zVUGUF+*-pVZmB`BusB8{W=2Y1xwwZ)aC|yWx=`1YdS-!l-C;G{`CIywmJ9P`BmO-@AXXUeNRJX zrJp>jQ2U;af+eKC0F75kl-1k)3TlAZOr>4>Q?l(>5Se{suba{)_p_Rw* zHsL3C4s>|CxXRn+#gTcldZ@`&tV#OTVqHCNT&1blY@R}S8}Rhn2@Y@Vs=Tdvx&gfT zo=A!lcX{)z;_KJ!Iz`T6cq?~q?ziXf*>thP+a*=r#`k|N?t7}4YBP9T-kPB+{)|8! z(pxdSl{+-N{pzm^_@0`-FZ$1IF2&#HaP{&!z@_-eg^VxO+I+yZ#%W{&7jZ`hxKU5u zdsFp-tMJwyyuJQh4c_?CqUKpkae88D)}og3YdraQlPXGac0wJ?FXdr8n>RW&I%5}x zw|1{Q@O7KF4&d#nLCP_;nPiiSoNL-V@ataz}NGH-XG&z0ON_%m6lbq~I z@?>;nc)RhSU;f$Vts{85b5$MQ*n^}qdT99pRpXKJYvNLVDPGL|CUq`#CB>yYQk;2Q z-lSEGGlQh*FKffwvgH zY~DJ9w@cU6;myj~G7Txteqw3gS{i>iZ_@5qom%6UJW^bf-{nop78p0Y-SGZr_OyBH z0^T;hT8B5^9yAm1%^Azj-qP{LZds9WjrLtD`P1=cRW!UEcG4ODW%G79cv~{!#p2&n z-qqBaQ;6jW=Z(|mE~tIGWln0}JnX^cjbHW6Oh5CKwc+iknY;dr&0AOSHu2`jyoF4O zd!d$cN^!20bU1HPX2#heH9fJkl;7noRz)k1;q7bd@4dw4?F#VLeQRXitR7mv-%Amf zCakGd3~!oF=n2^o^H3Mc+aBZpaJJ3cmEi5A>s~7UJ>@hSrJIwt6w@e_w}aPqo@w)T z6?nV@SqJ zmGAmccl$l<0p7aaRf9L4OEXL|RzJ*BIR!?fooYPN+GdDn^A>S;=I7J-SDik^=B+1q z+j8kEt?Pb^b%Mp*ZxK&3_fPuSyv3SlIgjD(U%&B7=jVl9;O({vb$H{i>9IE{o?^mM zElof4P-pYzTao@)6JSn_=I*HVys>jlMS98=4R1F-c-XD>{ni`2Ex)G@Z|s4)Xq1lC zPd0BMyNlJKZx2#kJ#W4hEx+OIkk5Roqs?0%@HXbMSBrU1rD<3_*L^iKc{HAMyrtL? zb4t6D;>GY*u5Ngn)NR~ao42dMTbC&jc$0R@DK(pJe%_?1N^xnz*}VB?ld6>ChPO+P z+ui#2o&VgXFL>K>e+1q#dZ>F}ac#uC*R7k5x753k@=Nh--Z+11DH}bOHoVQa{C&>< z|8@;{yQ#}-#k{9hZ*_S{?bq{`WAZJ#3t1a$0{M8$QQh#i@~%JLY`>@dz}xicb$H{? zAd=3}Tg!70gR$n*x_AqjK%u;? z9kAhOo40}B?d6%RfVWtT@m**h-q;m&tX=2hEeGM|emi-}UNdao27$K+AF07x$ga7T zTAH5ZVVqjZ;~TG>f+jOP)S8NtUy5sT`g&a6G}Y;$&Zus9oBsXPU2Wb5gSR`ozoCCm zb1)y}`@N95(%R;^-!e|H=Jcc;NqJn}BxhjU@OEFDnBm_0dKSBdhiz5Ps{Xt zCl;$N^E3}{si#pWZ$FrE)Oh=TyB53+d8`4v@!T7wi{ULbOPc8w%G+L#t$N1hZ76uV ztoK&;J&iRb=AqV1DYd6^3bDEh$rG|xGbNX|L>0|@`uehmoWGB27iQ(gR3*c2czf5ZNnf(x(-GipRo^$= z_ms67r8zOF2YNE*Q1j=M$|*>hrMRYwW-U9XpZTfj(P*yC=FK-7Gh1s7Z);C{&jy>f zk>G9Gk|yQN+Di*FvEuo7i&eeIJlVWSnD0!Gi(weot9@gSomp94DRgy=Fm)9EJzWc%VTyEcQW5L^! z!Ea^1r|cm_XY|>S8>4axjp~YZB^@&DcKAmwUEV@6OKZ!!FuZ+z%m0qDc^e1brmilC zH>uA?#oha@QJti&TMTdIT^Qaj{lcT0ZQia2Zv$R#0B?Nnq-kqB(ppVCE^nG{H6Bgp zzMevP`?vdU+1KW6Jb2qS^lkS&ZPa9ZCmG_8%Ueh%T$^I@h4S{7M|Q=^xhL__lq&O$2XCU(3l` zM%~=s|Ht~L+@;<)!))H@l(PB8i_FPddft3@WcdwmUElq}J~nTYz}v`?@3`-&l_xhp z&73Ejw`S{{{P^~kjyJ0-!`ml*c<+tqQt!?XKuX%X0Ix)OG`JKKiZQiDU zx97LE0^ayJqPF&1_a5fj3S*4;X4Ckk{8C(#)7O)Ize$})@r>$*x6V_iIp4oc1#h>G zzO>~0+F(Y`x@Q`)*7sLF>Gz#aKHgZ7J7t`jp46HXU=P{6u`8}(oS7N7wBc>c z_uDk^Ht+T(;?3$mDchA(h*eQ}B1v~7?M2UNw~*agJ=FYkhvbj7HXm=SPOaIs#>1S{nzfRrP~Lv= zqyJuE^EMN_t!dw(w8PzR{Jxi(Cz6jhW@rY@-BGi*LV5es?VDEFyv+h{GbVcQ#vMtT zJJo0@PpnV3)I&CJ>`KazBA%kI%Nu8_sYnmCH39RUE_v|HgKXYrgSYGM&B>cptLD2& z5BI2v)00BGyzzI_q;5HFNlV?*FL^X^O&)q&-uP8bMUAJN)9^O-jbFdZ=4}pm>(!}a zX@_`EB|@ZGxx7iv6mj2fLway|ld}1CW%&(n-(URQLu}sWg11+uB;?KN$LkLCT`SE{ zio3j}t|)bqI(kD@hvwvUhn^W?;yh%>q4x7#I^2QD$?%MG7%a_ji zu6@5P1aI9Qs=-^t9_oJ2#=UEFtloC==1xE}{|t}e?Jv6>`$%OlnSvacX)bO+PjLj8{&95jsP(l+!n^>47;V zoz0u3ipEo}Zg`tK_vfQ+-j;&5={-7^lD((W1T_8A6Ou<->)vmYlW}$xOH28E<5E8^ zZ_KRGQf4b|cpLEgZcA<6mVvkN^E`NCZ&s~_eD}hBsM(tocX{I~NlUK}t57o! zntp1@&$!DQzshXX^ib17UG^B>&b{>Wi*4SPgSTOid+^4(AzC`Mket42^YP|er5XI$ zyhZHL@V57x{hU9awF10#?$gD6Pc3e1-d*%)bgX@`mWEsU*+bikcauxaBdteRb;D&fibI61;U>RD(C>ku*NDN_CROA#G#>gj zTFRNtnZtHImaYc4M(`}s8F6hfx0S)0vUNaxMAwiw>ZGdK6!Q45~`Uz@iz;BC^gSXbpyuSII6cjv^YPYN zcE<@9%G>&h5B}BR?Sv|C_pR{Yjk}O^YdMj8yh*!m=6IpJ?Xzj{!wzpJR(V_eLImDI z7~?PPP-{+*ernCy=HxA8lG3jAym5xq%*;4-*<-$ayL;hJpLcj`Tjg!ykSj{+_mp*g zwPunUk8fScUp#M`-Dy11j->D8cvJRV^JkNErx4Or zYgwC)H|tysZ&z(N>wJf|ldHVF@^UNSE#zdaJHLhJ@cWI^rVi;g#oBbdS=}1m?)d0= zy&c|8sq*%~uq#W>dn!#&it|h*kJVetxpS@LVVr*7TJ*cT@vE#tEp@^;HFGi^qNRSM zoGx$7KrLmE;@n+1kKyf!{YG_gc>7zGw-q$+)H> zJ({)jglMU*%Ui4}Rz<_xrJuXvO^3HrtGvyBy#{X~{iIk+CS&E#<}F3VSe^UkiOX9? zb;H}K$1cG?pSAthdrzzKHe%FOCFecO$k}rLw02K6Z~Wv^H)_}Hftvj=PF*N(fBW8E zYn}V;^eS&XZ*66~#kzU2dE+@Yv(`+*OE#u5XO^>g3d1HoHRmwRHZ@cZa1%F-5_TOK)pvv3i`y%khv*unxG;>l* zIqA{NFfMOWS5lnaN**aLt(D@%@K&yFc)Q|f-!3`#+l5u$dUozvD&#$trq`%=V|nAQ zB~IuG(b>F(WairPXWZ~MfAzB;ba=a{%G>KxYw(tO0_?LyY?zM z@2PZ-n(vF~p_cX{c{FkMDfy+kl~Z5_$*GBJs!O%FmYQ{?I6aashPQHuhPNNxyvHpL zZwQgZ|po|rgQ0T2bpUTq&-UQW%J0P+ zC(x)o>>(X*tde@Cnp!SznpINgTuWW{8{Q^#y6z9v2eramNAR|GQw`pvT2fqk)Fr(_Z)sxhn5zXYkhHi5k2m>Zj#y$>`b36Y}cmdE+#wS)Dr8?qcO}d5e|V%42w& zH28b}V&89Fz}uESSC@j{Q=Wp=LkmAV5smiEBl#I;T|IA722DJLhdHUaJL+)WH0N9H z#PD|Go{!=WJXh|w%fZ_Niz4vGUHfWIhFX({ernBHmp5jRv?jB!Csr+qihR6D*)n!# zc>8}pe)>fFe(MU});$%Ow;X-8+<_T;iSBWEE3&$8t&pka9)4OGr)D0>Pfv(uU6(h0HKdBPmieWfN**bv zChqdaD$Kxm4sG7k3wHm(Sev(Q;O)7A*OW5fQ`Ti~8tuC_wVyq^yz#3VEp;o!Lvl)M zHMLycNINrYvPo;pc?@so&ilxzHgDa*+k-3X@Wy^ZbVi>$zkjPcfspCByoH=&##+PM z?WcY6T${HZ;BD!P9=wI@ntk$fUV&Z;d*ew~!rKQ!>1L?U9pyZ1dI& zyj{JvAl^dum11o+Z>(rdweH);wauW@@n-GL@HXQ57pB;}^#*Tmz7l~q-mBEoNzp@X z^`G=)^}MmG)ZH?_q}h)&VS4nuNfjAqW-D%Zdwjo*``Wzq0dLO??_Vn9J!PNLl%zP< zQv3S3HXm=y5KD6xl9p=GFM0I5Ng1TLRo(D*^k?^c!shL2@OINp4d6}czsPtvZ`QPv z-o_!5NynSjmErB_5zl?b=B+PyyWx$UytQ@NYb`8UC*Hz5`!AHY5A?g~Wt+Eaz}vv< z29yeaPo*g_F8yI|PLh6VjmKq-8DeQoC$VZtc?#ui#qzHmYxC9*yluNRCvTcbN*;cn zsnOCT8D}2WmEz&NF*7@%riYrI)Y|1Oq!VdvId0z5C&yfMkIh?u@b=WUoV=x;QtJHS zypihE-Ac8jIQwyV<5weU!`nNf$KoISsQmwy1HjuYV+WQp-&67|Y4$967^k0_9%_21 zrD=05HP=>7LHeo`X9jBesWl$@skxS#9%_212dEjO?B>+nwhET(d6e^Uq9DU>v`iUUtR9j@OI#YkI%I4 zx540T_=E_&`S#Wfe)oPO8_m#RNDthL%NxH+9a6Ep*6{Y#iAP;!^EL#$b-HIKZ#91X zEpYNJzYf-@Jfz6wEm2p7w+nxCz$G?s*MhgFI}R?X-&5`)QO~(g%C|T7ev@|On_0@E ziKnQQ%^NFHYr3^OhPR(wJ@PX)Z$rV`&67QNv+~zuBc$h$wVK}O(euVtMRv%#Av&8k zO=jh;4R1ZpI{eo*Z^OXbwEOGu##)lr^xSNo$|-P1>`;o+(^|C4TcX_=-VS;2GUxxd z91h;D?lPpLeoq@U;aKxwo_zOPiV3i*6!~4=e5*(uT5-eMNguum|KLaE*F#2tw^ygN z0^T?uYmYVGbo!~eHv4|#s@ATFl=Dgc}Pe+2c`yQ;p8_z|mCH?9U*HZiD z(fFkvq_vE@_nU9FSpHafxI5+y=Z)D|#nOhigMa^~^Xt8%z}uQDt}Qw5sfH7)me)Pg z3~OE9itMV$35WBhoQ&b^)(gILuzkN>2i_*ntihWyzZYXgPFT}_V|kN0OfjWGdE4`W zx12wJIvTw7eWVU=yi2J$F~&8TYc-muAm!KOp(h`2tj=tVvs-#JI%KVszfj(m?t8{p z?fY#Ec)P6o(310>N}V&FVlvDx=~#91@g{ZE%<*jAV$HKW0rQ@Y_<8?>Y~IF#w%rTiUc*Yxdn)B;53NyiU*Wtl zTQg6?3*+j#IcazPE=67^H_&S_Mw$|=NRCRUyl(9oqlvSY%NxI{$)@xe-oF3bu4mi4O$KlCp02~2rWQXV z)Z7>2)SCSCa4j`G)a-%raNac4S<%<;yH?8Y^2W>|+Nx-H``AHW{j1H}6!12_|A>62&93S9)y42uo|571knYD_X!AA=yls6hCvU#H_Vvd)1u0ME6!=wY zO}FHbYZ=$%N#S8X*}Tz7Ep?UQF}yv!;upPa-ll`MCkBlysozu9qLzB|jWef4`{t*= zatiIRcWOE%%?|02;`B&5WG!ox_m!`|&*tq0@V0SP4c@FgEpH$D z3HcN=KQ;5G)|FH6t)l7HcQR5==8-&%Q*&P~Z_G?*pfW}xWv3Z*T-qyWTgSWuC zHTi6|oGx#PPND8JQrC5POLP{7w{7P?)7Iu~CU~1VbW{m%ihBx)dTTia-)T#JDK7P& zZ>PRhr0PY+3+3(D56tOn^EL~-UA?Zsyrr1DhKhW=rRbJ9HJ#J1(S`E%gU@$#{(hv{ z;BD)x9=xTP1}By}KWjB7Z|qRgzEdicw@-X*i}U^49PqYr#C4@y@2T&uIlYjz%oC#Z zyfs6&;X5UDA=xt4n(yEKd#}Os?Dupocw2sRPTn%=wtR0U{lvtzn$wi>Fiy?8s2jX=$w#m-0yQkUU&V(%fCsNiB7i5jVWef9;azZQkaAx7k~3@TOrPqizdt&Fra6 z-nT=|WHcV%x*>j-w~%ZZYYlIE{QIrD*}TmMZ-YmVF6DYpGwSZZycN0YSlkxM+b{p= z(BIj-EdXyFZjZoQtjQF4n)&XxB0FTQkO>#cTl>wc?z4GY2;QD}D*|tvOl#5Myh#i& zPTi=<6w2FxQy-pW^R@`Q-861Y$$3vTQ_^@sCd}_$B^{FA<&9NBGD~ZH^V2U)n;uCQ z!&|w!`TlL(OYixN&D&z|Ht(()yh#)DjZ0H1GLO`Q%UkNMq)vS6N`5JSF}#(#GQ8cm z-#yO%Prd}a4Q@ZS(V2|>2Z1E zS0$~<%(Z2|;ceBr4=%Ld(`Df8@_Xy>#`{6i?5(xveD@nC;Jee1wZ66JcX{JiHCoxZ z;q84VJbQx8+j8*s^GlydsU!+BFyG`v03`E%`T z-d2FOo2Rs5-js7l#9IhUA!{|eX8uBXD}8LiA8p=Ng16}p)ZmTpvPMf2;98A`en~Tr z#zQ}~{(fT>o+IPbtm3PsJW`zbB~LbQ+>s`mRHeMu+;43TKjJ$!Z>zxD_{*;^wW{}2 znh$5C(R$uA*;07o?v&J}=(e2G@OI-3zkAZ=Z8do7c|#4}V(m1=zGC&J=Pg#nkZFYU zK+^QP&e5&mt=;LzA87No2E4tzu?BBBYSsK6shRV{oqPxnnkj|kFO;_rPCa2o%G=0C z3*s%siEt*=np%3^d^2mR`+C>|wI)wCZ_Gxm$!vKHZwFq}?N84Ac4C#c-aRIilD(%^ zKho!-A#pu#v2I-{j}(`t=j+erO{$`aTb&r*9{loB{O?HH@3*#9-d>s8q`Yx2)S5YH zJjL^->B`rmnM^v~*loEJ!&{qm$Nj;%-%hIXcK2g-c+>2Nbu~I9XUJOirss{Tnn7zO zKu$0T|Z&uDIoG_Uj&PUS4Z> z>oR#G{&|`0yq#9%ZQT+N-nfg@I%Myf-s18mRrKA3uV1RmxSltz%Ax?sdhwHEzjgC-FD`HVsx`^FnN#~iddtUKjvX1^4j8lh2OQqctn#*Pc^%$ZSF>w+ zBrWAk?PuL=-ZY&s8#UKTaeAmFzZBQ3b$OFkNmUq^?tyYn!`mLOzx!y1x3j?8^B%mh zu0~6}Y1XFpR8FDD4vVbC>AAe|s}Z%~ZTAN|^>%nWyUN=`gQt`--_wZwdo{;=f zPAQ(vo3yH#|qI@PNqn|p0^^qN>Qs&-hR6EJ?^iETu|lh zzSnm0);1q|yqBm`FcXrqP~INvaO9tz_w>RlZ;M7wD>?5ezmK90*>%WT_COtyGaq9i z6&tm-P~P@myxVYxw~MO0jlZP^Z;hG^>uR{7N7B;GlaDv43Oo1J?2!32dAL^c>v{97 zZdEitFMMR_SMbjnZU6oHi>ti#dow3*+$puyLzJHdQsm6Xo2C=rX{7e+dDHAJ!((`x zzyB{@aqhQvRo=Fao?dd^(~O)A=g0SSq`ef+TddBd9ZB&*d3)dYt^;n{@3%{;yluR_ z25*{sRpXH+AjO$e(p*cACQgsb8?#B88K@;cQZ(K#EZ`@jI?zdyl z{lp9a_t4+}eEp^P`y3v4y9RJfq|wqiq_w_&DNjh;1ui68Gp}`lOVq6a?u|izy1NQo zg}3(LZOQdFl$-}u0$u8peroy|*J!S#j>T$o@)m1%oN%GM9k%vK>))?GqD=?zHhFyw z-gv%})=Wm@(d6g!B)@MwA8#Q$Vu!vv3aRVz#_H7UP|_KGbH6Qn@m*`}`>i8*>(^mM z$+_R8T2ee@8q66h4|7U7A8*X&+pVvkwIr>{>GCFJlj52x<+X;l|33TMFWS6a2HxJ7 z7=bsb&t{H?^Ojjd5&xVHvxR-U@M3+Y)} zn{jVP=||5SJE4|3j}`Yl1@=bm@)oN@<|$V+y#2@LZ~mIiTW9dLs?*Gp^PaL>YUYTdYn;+Ys1?S18&FfFjRQ!3f?+&omFz))6^4ahWy3cZ_Tq)Qpc`* z*~{=7-cGo76Mh4=!rK+#?eQBT^On(1%{z^I_thb_S{HBOJC%6j%%v()oNLS4@b<{o zl{ebozg-F5);tuMH>-!}jK#{I&6{$jvJ2&Gz7} z$SFy+$c*HX;#^xf1x}J$%D_1N)buoic6mDoH;kr3i{8>$)j2 zZ#+}YUg*)RrH7h%G`g7kP1A|w3E3TUy1d28%qnGVcw2SIhZorQTMzKIrN^8SXm9>g zu=d#UaxpH=lkpv6r`)xqUEU;TVBGNb{)4}|)aI=xc)MlZ4#=A{x4679n^dtG;^}y^ zCSZ8`dn>S4c zb|U%9al_lMFYNJAo44NJZA72BCFecmgjp*@Gbgo_Q{$1=YU1JVY3dF&yP!wvC!04- z26{5I;qCGN?6jxNTOaV&VNngkr;89XP+#(D&4Ls=j_{KF!aY)VXiG7~aZV8QxYentG7U+W_#k zc|{%GVx6z=lw$d%{&(^geFBEJPrtFx;Wlpr!Q0vwYVgK0;&&91rk|Y9&p5T@k#aIF ztz~{m>!;0%%qGP{x{}timgJG*+4q~YiaVv2{N)Y}ZyoM?<*znxgTUL2Aqz?keNVZU zBDJ2kBD+daE1S276%B8DkH561&D&t`Hh9fW-g5p0mmw#``zj=lW^JLo9lrX^DK>9I zz*~n`YVa0o8XmCv< zJKxjF1bpWv&9fXgynW`d^V`|H4FzxOZrnk5;~X?I+{s(?2^ik`zq;4$HgChg+qBo4 zls7U%O|C+8zG+LHNb!&kW38?1l>hb3tlXX9?V6vS==}fW!@=9tqZXA4c~3PxYdo4+ zY4YfK(`3_h)hG{1)9*S*w}!V4bNgIt-)|$p+v~R!#2fqPui;SBlUj#Qz8yZN)a;yb z?v$E-DNc{1S&Q1`O{yZrS&^Bk%O1nq$l1T2X!AA_yuI>f4cpET zfLi01JRxzHHz}K@TWM`MZg@NG#D8~wJ{<+#){j|S!soQoTVUO~KO1KB+_F~cL5edc zJ=FB1oJRNV=j?Ddb-2-2DeqI<0 z-kw@tgEw}*qv~wl8ntWd1Y5qF4e2KxZ&p`^x8J?}i^J^uZ5(*Jy~9%Xeq$bfQmE3o44!1+uTVtcw@IICesZ0 zUEWgcG<8mCl8mS0jaAA!HM}kQ{ujPs^EMv54Z5!eZ_<>k{84WE6gj2-8FzV0(MgJ& z+=WJ`NY+VgGRCV;oMJ1r|YKQCB$qHxu0IbGgTci1TQ3gzvGPu}VLeKixo z+vce|d8_gNTZXU^vR1>qZ~j7gJFV;fpRw<^N#JeGhMm0C;7xPxDLlR`X?iG>w^_%m zxW(peGI*QPb$Q8oPx+GfMvrDKJ(8}R0>8>C8qKxTnzb5FNKUSGdE-|l zt?AJ6nD5`l-u=R^Hg8kF+vpiJcw-)4#<Wso zTGN#^0mIvyU-_N{PS5b#lm-0w)-&*vi0ao-bd-fjSI zOE*R4Ek~cZnPe^Bxit@O+(inV&0EBZhPT~Eo&5=$w;AAVY|oV?=RNg3JNC@A)RKpB z=`~@Tnro@)p{6IBH?H#C1*=dqC*v9|t!12=IbGiPRca}N6t~tI-uji=e$(b{CV0DM zehuE3rx~<#;_OZ1p+B29I;nlDaBZWsp0~gb4R0^)KmApkw^`ur&Bt2-Z-H~D`CjMa zEk!4(a~8_m9q-%Z{Jbz5ygk}yRmpizQ%|57@@Mnb8t*Y_s)h3QjT1Ni)V|;5fVVY^ zYVfurO*{J(q^~wxJmf?QbuzHZ7z75^i&Prnr+%C`pLfEQdCTxQ)04E-nO>A z|NS;^^T1o5eydBZ_C1v*tU1|i-ZYst9!)1HJnSu;H)dnER^0r&@cs2ezhd(?AH2Q1 zG$(JY#h;g{rQYbFreC8qYb8(R6r`^*uGuO5>|7IdDEdWuF)8=gfczgEQoV>ACYtbYp`+f`A5vzpciM7_{jhQ2MYk0fwyqEuC^R^JY-7#=Y z$$3vB_E7WJBDmj1>2$m`YDZGtY~Hv^I(I3~wPkI1d*uE1JAdAJ5qO)uvIcKfo~C7n zeJW?tvX^wcNu5Y>?n0w|^XPfgRL}4j-hQy<%Ny+bZ83No{GtbMzB|q6t>xSsX4lIc*{}U@OIJ>KYGLFZ3%dLXUN);^PYz6uC=Vy^Oj@s4gWOp6!P)L4AdFZ zFuZ-_zxDGox-SAzP(4 z=A^dr*X5+y{M<|P@RoW?E^i?fIYTRt;qCrW?;dXRwi>+Mc54mZtUL|pDr1kid)vud z+zFJ=$nZA*-)0|c^R@=O&Dxfbw-lW4vk;O$?zGuev+20^8+T{T-|+T{&(8W?mA8$% z?RxBA+8lSxl-}!l%sQfcHw+qkQ^|QwnNs?;k)^1)R*FkGHEZb!(JpT6R?<>MX-86A z%HtcC^0WVJ+?at{(+R7T{RX#{L*5)v#ch`gZ^u`8Yky}Q-ZXVvqsQf~HSR*wb2e|9 z3`&pTZNwLse8l1Hgeq@OzSE?2JpDg^S!`q2f z-fkIxb4mT4^1hX{rf06DUy9QcOS`&VBpL=1t0$l14tW0-hqqIzy!E=j4sX(~ zrG9+<%)`$M<0*9W^CsSTA?R;y=c~5=o zN|Q+umwJ=pQV*3=;3{fq731_vn*K)VaNf8(X%}UW;ce#T-T&9Q-%hRawtiX--pDOA zYe{i>G@5y+HESi0CQs!Q=%nU^HF0)Ae@L8brGDt4W)IoC(J5un#LFJT+p(t~JjCJc zv?^~mK3Ib{=HF3u?6}rmpn4(5!6+ zkIS28N0NteYVK5u)8nf(IkS1wWKecxc>Da}(_eIWJEO|msF^u=)AYbsK;!rIXl5mO zT;4R*rEHqDAs$U#Y1-Mmu`5j#`YmmE+jim7D;(a=tn$|VkvhCdyRd3S`TeHUwU|F2 zZ_=)rS@ML$ee=_==Pjhe@>;{&p=XZn?eKP1mAB`*-CoM|p0YRYj`3#J*}S#J9ZAzJ zl($a1p0uyS+u2p#?wnJDw^-9|2a>6!$w+aorRM!l%{+`#yS(wMtU@hiW}I5eNe{KJ zpKGN)vw34hYG$UU$I|A0oBF4#?s0fKr^?&?n``jK{LQRGJ{K-;Av+D}#J8?*t!&=J)a#BCZlSoRkkMO|@ApFT|D)}c)P>9~W3AzB&awYI z$GP9mtMWE_K~CO6J{Ov`)?FQ)&urdUU7Bz+>_X~|agtWin$DR)qsuuBZ@=x_VW`8~ z`BmP!Jz0mh6uoKsPwmO(Ep>I@{2`~{^5$Dz={LOn@Cy$d;P7@qmA4nKzO$71o@zen zntpsI=IfVwVE@di=ZzUyg>h!4N77uY@zBp&j7xd)@utaaRW!UE@`+FOb$Gk5%G-v; zb$GLKw!CZXhg$Ndj{EkY=S}LI6{#h^6!%?A-k4L0=i`l;mD=$39|tYMAFtp3^TI_{ z-c~=IlQ(4#EqpecKOb*AQ|e~h9s4Piw_l%qz{Sq}c5#)rN&W9CWxl8EJcZ_`o7%UQ zv{s^`atiz^Gf=aWSaIg0WHBDKp~k=2W= zRVZ&C{nj2ocX+#`%G+hn?|{5%Zi&h%6nXBAsuj+gG*2sTzJD9J@=IO7+e3f<^YxeF z?{j!!(E3u;2UWV(Ic}@bmT&T;c&xR|lMS4v6J`t1oEUXTPOc3Hj@dG_0dDQ&)4o~- zuEJY;@OJa6I=qEUz;|tt{>mvx6{WZ|ZSu*r)bvPsQpB@)V~3jR8M`pNweQsb^EPiC zz}teC>hPxN!PleNBR$koEy=?;XO(@w`BstqQk$~a z-a3M}aYOGeMSV||y|uidlAos_c|zj(cw@!X9kN!4E|j;2r~Es9L8S7YUIyNJtgFGB z?<9Twn(6uG^!11HrrAX^c(Qqu<|f6tBgjK{9++2gVMEyiR5o^C*`HSI=lcd&k?(1hxu622%Gom)UE$OxA9yV{6 zgSSCjYw#xZz@M0?C4V!=)A7d1P;(mOftsFJTF;x*t+ErtTla%5KGNo`D|qWP`rcCE z?`i4@YvvQqTQlrJzf)4DS>@{~Ps8wb>a_h&uz9-zyluIy0lZ0jDe?qe-uUUJmbz^% zagyflI^r%2Z>699zPruamEi66x9af5$xB-5S@JW^wTx5KqtRR|>B{7#uQD!Gq{mnL z=9K*Wgi1ZQywSC=Itu-HtxPs;qNK?kEJ;WP0pmBbg!RWG~N2{ zjyYZ4_*F^gsAzcW^8RnGvw7oy1cOpHCOT91f(ZJ8{R(r{G^L)-g<$z8!vmHq<&8|Q(`~9ey&Zevw2Hh zy~zA}-U7Qaybb)p_%GVL^#*TKr+Dxd*iZDI+Cut?%Ukn*vPfOX+%nc0-uCEw+gh8q zKH#nY1Ce?2oqTG)G@rP7iytYK@0}Niz?%)4dT@D5QKcDj7RuYa`-Wa`-*5fE+srE;EIIFKiYfWBr15J!(q3HNLb6F! zq?Cz9cc-K-MfGw;^PaX{_`xb} zyHx&v@&Vv&+as-vH%>BSZs~Z7)nUl)xR#`)+jY63;jQzkUib&aE4&Q^Z;y1}Sc>|d z`hGXd$xz42Bjv12K30{GJknamsktMK&gM;%nI3AZy5a4Nt0rDoz2GXm4FYd>&8@?m z^l6piRu9pCrjYXc_LKCJUf8asX)vD6oA2D1+0ur$8Gl*&O`EsD;BC%hb$GLK)}5qp z|Gs|TekA`+-l9*y@b=m-{<_HKZ3uX~w)aD&T<@vGi4>R4T@z1{Gy1gaei9qIQ{Hl# zPI7n*Z=e6}mnPV}T?^jYFU-kXj()sOBn4yaxibIMRanti`}QXJNm|lRnc?l*^Nw(S zy>}>h+x%o5-uV3?weR#|`Pl=tZ~pMf`&Re$`|e2cXY1nWjyij%q>fHa2FY)AXn5=O)$1Rzc^e7d`VM%cpEX67o!sU?plE_s+gM3c1aPno6@dZ^1C8s2vK=Z*_(-bR79-YaVG z7P32OEk6s?l0S9aoxIeQ6c6bzR!&VGJ#Qg9Dz7!X-Sxti{cYZ^18*-qUl4D;d(li% zei0T{dVKJ8$WE{Z)3pQs?~LPV@~$NxNmR1 zYo+{B9`-4%&3C_PGD{U1mpZ2>me%v8smKX%7iDdDd+&@DKeTxp3*M%_T!%OI!2FC$ zTI1Jvr2LHg=F#)UDx5as)S6B-c_a_>OPXu5d1D6dLZhv9K87ZQjO%w?|)#%v)?hJ3I?eT9_+_wMu zg^A#8{4G0qYg-I^%3C7(C$~`EzP|iBCsl!~{Qu;Wz+3M(cR=1WCtD0-%9|lNW2{>! zZ%2Kr_iHw9lfm2TqaSnMQ~sPw&0Y9v=AoAIXgtzdO*|iOsk>r+YR!%$Pocc+v+F~< z*}P2wZ%^D_gEy)39VwoVx7Ij;e7tGi2x}VV`?oi@4%=e$HWj?x@^%E?*jubmr6vzO z)SQD9ubhJPRc45#SxeHITE1)Z@y1T1iq^Rs-tPM38_vHkGYz~gz5a3cJ!LIw)@9sR zvrpf(QV&vG${&8eX(~z{#;K)lHF4(Qg!A#nY}6T@7~T%)bR_ELbZdJo<* zY9;-SH$~lW-ZWkD+%^6bo_xIJ=)~~$hp7jyx8Ktnz+3+gPq_D+@029|rMR>gsjd{4 z^3yMQT;8}!>QIVHRea-`{8CP?{%rF$1H4@}sRnOSPVPeT zgv3+lXKyZV{OXRRUEX|mr#u(K+vk4xW9NRG3ErN#*Mm1rpUU2nzQ39tcJdZ|0*1G? zFaPdb`+l1R-fr&nWT|O>KTG07ifj0nJkh6}^b;iQLK6?a-&iH%EDUe$j{K|h^XY8x zHf?GHcw^7h+*fKHm$ydkLNldo-q<;{CNn+dwT8Fbcf09M`+l1P-ui540B`KGQ93Sf zjoO8#ZZ>b43`&pT?Y+Oau$#@>T=4ehMBJ%o41I&Fud)0+W+io^R@`QUA`$NZx-8i->(#t49QtJ zg^&!IwVV>Qw3Z%f`WdH|JlVW412wZrn*OpjygmEHH4odoEe3CI_ISFaeorHQ78=4Q z`!9w!>7+Dqb}RWco^0NjS(A-^OB>z}+}-{AqL+ZT2j_Y4X63KRjBn4reoYS=Pd?r> z-D*6V4rArf^QP%a@|3$ZyiNG}X6N7AT?*b7J?_C0N;qB^|sMEL!oRjcOPDtVZPTJrnGnUk7%;@)pk6;_nA?^>RSl#}sn-Xc~v zy#3of>yEQ|TLIo)+){%#-yLyxnzb4ab4ofSPh8%l4pYZD0czIG=8aA%13ekq@bhHcvXx)*Zt#vLj9@nS4> ze)oDyT`|R!G&x<~LUx+5*6?=7BfWb$yq!?xZQ_vUOU`pD?SdD%q@}e*#xrKp!o15H zJLK*{;#{lIzI8P@UEVaSa(E1HyMO14zjt^$vC7-vH6Fa>=%M9&i(DJdTan$SsKp5s z%3GU##^9e5-~OJqt@76Xl^ujPPFQn-+4q}oHcdr(_{ho3c)Y49+IM+5xXY(d?)yzARdd|mNj-49b{^Q`juR6S)T;*-~ zjX8PC(NE2{Rmvm9rK$S*^YJECZ{~PD-g4~7@HYIXv+>Iz+uzeus=SSTy#{a6cdhJ! z9!b+LX|AP5iqqrE7;{!mfe~hww69;v!?={wH_n{Y;k>aEYE~(0!`r7n{=^TQ`|WR4 z-Ug3)vE;m`zUOGwt$B}ycvAEe_kQ#3(AOU_Jw0!lt};A^w;#Uqo%0>uPOb9x)-5%7 z3)x*pKhZh#?UTL5<;}N~BK>;ae7oX2%WDm9Ck$Bp3x~JUs=Ph?W=`I$T2WX^kyDys zHgD{NTFR#JNNZEa^YNBrM~1gy7ykS^4sWMdd0RT>rIPcWY9^4QpD0XfdXqftIh!{r zLo>v=3r#=yc++IIJchUDXFs;Y;q8nnZ)5Jr$(vOxI!pY%lv=8*@o;VB6nxK)86@r7 zNov2#n{P#Fr&iqX_OX-N@9Xe(W|gYbKBM1_n%=T`%EO$obfLU`aPh0|{~vNrmA6|u zyi$sKzwzdmbURd}mi&xMr=an0ZT9^pWz)pzku*CA(bDvoGn_Ymnply#yi@a@9`=(H z#ya=gx!`S*2XCy)9r@~5d3@{ApUoScjnc8E6wVtbkkOUl?aSYtd85PIc~#z~-&cb- zekxf@iu-yre%7MS*l*461G$!UE2qG(Qqw7EcH+BM%E>$$%{-Dnn>VhKvN29=tu?$o zbi<+UulJr`ZWcX+#~%G>>2Un{BK(-ad@g%y>*bBR)@YYE&8i$8!`n&!{@&Hj{dP%}x1QZz zFR9-8 zK~>hO$&c?^U%$2AnmtIKe7uEp7_wH{g(mLumZ%fM+lDp=JZIl;9l_hs`E_{H>?p+J zyZ2cBe7uG1v{7qa-V$|Uczd~Ylf&C(;H|?G4dyM;-s|3VJ9&#f0mIv<8xK3kzTY~5 zw=I3PmNMVdM)8oku729SH)`t6nP1aezWa@tC2e(LcspU%%m;1WI)k?x7wzP&t)4wj zxV7kT-jq8vynU|2zkbB#tqXXY{#29l#?O>;zb$X5X04tvO*Y>Si}dH?Ek}ojw~g)h z`IODu<>0Mfzc)*n?c>B)pzHI&b z&PTTC2HrZpSc5ldl2Sad&xX9Y;k*^O>yXJ8%G=<#-?P%b-@1dh$A`RCQopCECmb?W z-`*;xz>3tq*+TrO>*{$6?8Mw}AO6PelWg94fVZ`4a`G0~Thcp?y-|~j?E6hx#SBGi zO+S*y~x^m7}+o`;B?1H8TnE=y{W>uv==$uZc4c zwPr0nT5(*ZQcM1jxXT+WQu|g3S<70n^5o-9Vl1O8!`rln z|A4=+rt{nVV z#d#t-d5b;)!`qkk{lzMqw?W{o=iPO9)0`d8u~FK$&&srYS2b#PA$1GA-#YY|wZ-Ob zFnD{pLx*DSH|b=hxbK90^JIUzeY5#?o7&I*vv~{YgjLFU%>DL}O{f3V=4}XgyK7QG zy!q}cJ+hezH?Pv2g6ueFB+_9MVG|?Wt7-J7E zZ~UsSmRR-8lfoaDw~U<{-d;Sp_s?zKhJm*}({}Q<^Uv4EnSkMK!AB1pYV$T6yuG<0 zA#WLjR0>CcD*e8J@x&L zf`00Zy|nO2KQ-4fPEAiZZ_G^1D!!U|sF{axjV5W~yJjbrHoX1vl0y%%c^e7d?wV1D zH)%&!EiWFpHpk~EcP;LvGI>TM&2A;FSsTma+e1FycrKdG%Uv1Xo;&*$=g%*U0&llG zT!%OAjy*8m8g=CqcBCCir;v{~DO<+G3~x&&Zuy>lzg-93rg!TU{+>#`HFLaj3aLBH zIGvVTI#v%-t+>2pbZdA!q|5phHgBWBTmLyZd6RZ0#WVVhK6y>8VtCVZlG>BaTg3A+ zysaGf>|C3-G2rd3O*wf}zUg&Yij~Lr)bzZ?>X3PSyCoG;9x3kfCOM_JvRlJjzdeq8 z$L4J;czdpA=eX}FCnNPq54Gf%;+nORN6(v7U5ZN;C4cI;%Uhy~hPMR={qjVcw{hU@ zw)r)9Yqp7{m`^(1e7j9O?R30Zog3cTp0f8-HgDI1x0z4W;LUept;NszxV-VJ)Y9%W z9%*fn@qE1H*q!0+17n6BWAip1y!F4jOWgOA-vdx{!oGW99%|f&llI(|ZYI=&)?)|2z-VB|y zPfb6?@K)~1@HX$UvCjQA5xhP1R1My^3w~0l>EQ%4I%KUTKRr^<`FLaI)}jmLZF=Wk z$J+1dB=B}yzsuvkr?E~!!=%O|_0ycZX?8@9q`B)tc{^j_PcE@}n+)EjEvvzsW*6KM z{fvib-<-bw;(7D!kkboUTMTdI4$XUd(jgzgKk!`n{%s0)>-Ag>-ni3d(0miwj})he zn*PcuFhZxVmhyyf;@gAdFNU{rb;H|U|NF*1_Wd>$ylorURsWv)PEX_4c%(^c)=Hl6 z`%TKsZZ#hIQ|N5oB6etaJ7@2`=i9tZ18?`Q^x(~R8nN~gl1Exw3~#;@OX1Jv&9|a7 z&vG8a+j~Cw``_EVO$ToaUQEcF?_P@Z@8m6f0@isM-hMrQuyeoN0NzGkdxiczwer{f zz4&JKN1t|K)5y4CYW5KKe&d;1ofzI8IOXo$?E7s7c-{-Ua7$wf7+tvt{JDMCq#2EQXWlRdYCi(eq&~8 zR*^dQjcamBJ?MEm2Y)l0)FG?TQ;r+nrhfN7C)>Qu0&nYwUs=q1s+k!3Pp!jw3+XUq z8oq1wyh%G!Ry4e=>Uo^=_an^)Z*y-dh_}GKM1QXv%bPTf6!CE0lwBF#zS#b!*Vy;l z9Pl>$jRy0k+)I=5CfxwO@f6c2l(*iWAMO18nIFJ`BO}uIg7vF z8oN_cCv_slIkyaL?zgMkk387E-{yh0mv8mpjs5s)_Q|y&ajvD7^3&t;Cau!MB@gd6 zdL+#bsU^QAPSV_6vkEmcGfr)J3~ygvclc8xyT^9Reou4M^?GOd8KF*HH=8$hLLI9sO-_2K zeS0XBw`IG2|LrPp72Xztx0QE#@K)rfRZ~mj;pxzy%^O#x*6b=ptwMSG)~tu8+q^9T zZ{sfQ?%r>yC*Yf3>_)8C!E%Lz7bi@{sh2{m|= zCXi99?iyvOEkIe5G7{u;dTI~Hn950YPsN8E4C?;)l9l~a(q;!dRu^l0+X@2jQ! zj7xpGy!lqApB1Ug9>ZIkKkoN!o3|C&<{Nz$+?7ik)NB>UVqEEo^_Swt2b+UO|1>QD3*a~6~jdISb`& z(S9GEVe_^Iyj?f5Al{_ONO4Y8(jjZJd6O!ph--GC@vt}g3+3(6Q9paW%G<`>c0Kl} zHpd+^rT4lXvyLdg0G%GGfg2ldjhe%j`fncG*rBF#`Zc;x+}=Iv@SCc*?NZ_G_$qH( zy7hLSQ)w3|rXkhR#C`jrzd3o6cFoQuj}#B*OCscX6X^sbP)-Siz z{RWwSjg}_HwUWm-UO5G3mb8>v@-yzcR`N@6)}p3An>VhaW(JKeuQj~==CJc-JG`A( ziAEt@1XhS0DX*%DRo> zggu1le7v>BUB~4uV*-EYZRvL&cX&Ig%3GfW9=v7L^19PPIN{ome)93g>a9h`W9B@g40c6m$jKPi{18{S^}_&vXJcssSq+rp>o@Fq=w@ra+1!fuJmDfo8o>*tgj zrCr{nT}W|d=Z3e>ym}V?Iq~hk-g{b=x8eQ!?tD+f?>gm)czv3DPhax8yrt;2oYV03 zu1#ki=-h9oS9$BQd?#;hUEZX#^F1j|9*sw;t6A&vrdgGvqT%h}Pi)3t7rp&{JEO|m z6VKJ)P1BFYBlX-2ahI`X=!!kWny||ot4KOWb;H{iryq2VbHANgW`$|-1^zSSi^YsH!Xb4qm! z=pWo-oBq@6MzYr@PKqARD+Y+Z%yNLpJcZ=(-?-@Bdr?Yt^) z_rF>YZ_+dx6|bB^qpBBKw@}{3Px$_5hqv>qye%8iKm0vyjkiSWy5B-3TqtiFkL!=W z-g5hUdO?-9u{THHt;kPntXl5<7ILP(YkmD8yLNetRh@at+Wfrm%(pLce?RNODsNr4 z*5Hlbg;Iy?I>p+M-s1A+yE}GH%{)9!#(lNR8!J*vnI(@EH@uyC&;R~E=RLis%G>ML z4G4cvr9O*{`<{-=n{T(i6@Bx^^1Hmn%53E^yxsnT(HA?sU0mgD!)*oe7VC6kw_7vsQnf?yj@b|?S^p!cZ~NmV-79M(^EMG2$(@lQLV~ z8sJvLN!gk??((MTmUE`gSZjFecK^bk+q`uIZ_Dqg z!&}I=6kMy((q6ch9>%HZk+hySX4Yt_q9)G#^fT_OUEY|1RjBEq&hQxC-Z|i^qixVN8=UVwRWe`m{*l&lU)0=VEL}N;SXG#ZI##zKIbGf~UFE1~cIrm$cLs{p>`F(?jj+m)7Rv&9{1a8iu!h9$4GO z=IwIu_U!aJym1%QoVIUo^iwmxq<#HT9+x*}=I*2nA#v7X9>%2}q&V|X=i`kTB+YKi z9>d$`|F);|{aaV?cHhRFyh%MYTUee0B;Mfyf*tiwQ{z6uO;nH@~4Qi2Wr-0oI0B~X3%J6*7$urA^8jCZQM;ixBmUp zquN{v-iFNb;7!`Kb+5KC$*F2QQhw$M(d;2EZ|sUyV#TGLn*4g+q$ZlSylc=ta)VgJ0) z1H9e#SV6pnOfuG5-kJLQjjO0*b>*9fJy1(|3gzt^{r+^a&0A0KHof<-?Dv$rkUl|D zTw1H~r(Ub)jh&>>(li*S_U$1qZ|sD++?9Dx-|qA2hi%?^fw!>>J$UooX*2tqhqu&I zipyI@w}!XfemmvEHgCPbThA>ucq6xx&Zt}SdpzdxU7Nz6k2mT4;_j%W&NUust<(ek z%wH&P|MkMP*1zw3RGU8F?d`t9_3tTnq}jE`!<-uJn^W@V<4x0{!rC+0&Q}WY8E%_OzpE{d2W}s#UN&EVxJes(s zmdhJgQL`fB)EOSb+l4^~2tpO=t7gY~4!zaPrjjXY)p$VacS4|OI^_;<&oA(aVZbuh4QxNJHNTv=B+ZAou<&SdTY-6E!OTh`9gX7)ShQ7vUwW--rgEKviSEj#Z)skO zfpM<2IyB$E9dyp5*X{dl5O`blat+>?leK&`^GMn^XDq+VnaC=$EwQXPnyQjbD{Ct7|+V{*blo&E<_> z)l`xETwB(Lx3){?zu&&!hJv@(MqU^Gp0X}`qn12UoNH5PJ#Qf$YVORC-D&b?^Tz6$ zZ1hu?*BahdOnCE5o3~-$?ZI0j@W#5-JSl2=eD}ya)XYgu54Fo1zsd})QFE$#-a>X- zo}S_D)Ynfr!scx_cw6^oLf%69PrcUVEu6p-)A7bCWo_=aE80)*X!AA-ymflVgSQZteAoK=HK#*Qy8A74 z=iG%xr{m4qsp0LZu7~3<#IO8*)^*_R@$1LrdrvhxrN`95hTuB6lPCRHpl&S~p; zQf8|Y!`lxJTe#WgZ7g^jHOYfF zD}PiW-of_WuJLW$wvUwW^-mbW>4sR0MQe4v;*HUvWIR!eYrB#|ZJ=Dz4PYUCbU*q9g=688xHfm4cq^);Kf@!;+Csdadh z@@wMklUlRZ*CTN*#a-Scr*9|C;ODgS@h0seV@ig%4?cP2Uu@ncfVT%X)ZmTXN_92y zjNWSAlagQRNAgR((O)?QMz~63WJ3es2Ul+ImRE5(^p5_nraqafa-K6#pwCq=w+3YyG~>QK|0#*@vPCWF#rc)P3Dx8G&+ zHW|E)dpIF){FEtsZn?W8zi&?Zsr9@`o%qIC-FL0z*W@gex0(O*h{Rytx;XYs>?k2cw+{h9pltx&)?tEcOL9&^EMT{y*9hSym2o@>TKSmPOJ&n zy^WbivsUs*apsqFI^L{`hPQwD)n5nMyiEgd_iXauO}f1qr)EElb9&VDNNX7n(b>Fd zx(cbT*{S4_>eA2tUEb)VmNF~jhPMT$Uhs9Bx9Q+*P0tDW-c#;K*^k$4kUD2JZ>g(m z@`p^AYhB)0ola`W!?>jlZzqq~+xhnaZvb!O=jY_jsui83z`fV)rx@O(U8J5K^Jnu$ zXP`E`9oXhWciZ>d4DieS3e&9$01J-(W? z*c;<6Z~Q7NG6UmEZFsxzCx;BQd7BB|w)L5q?>$xa(85fSe$JDcJ!JD1IQfR`n!BTR zdE-|jYQx*Q{@-b5^EL~-ZC+G|H|e~jK7IXC9!;FJs5NV|dDCPl(xcg(hPwSYRGL6vR2cZNX@^3~%KfnfvXG4QF+*d7A^?X7-!Z zdf(HKQ`7SnQiYtv%EO!*T?}vKP7H7VIkd}RHg9vm+rVWWyrq~JUjb^(BReNcyOZ)1%G)V>etU?`+dS}g*>fJeX(rHGJmI|YtTnsV_%$A>mbA7|-u`>| zubqD%a6Wi@eBfmFJ(W6_K0(q17-vrAku=waXy#|0e7s57m|3Hlhm+B)rH6T_>Cy8h zWni|9Zq0kzd-opB-^aB8yxqC725-#otEGOVxNrVgewVjq>oD$4NnNbW*2D~NJx_k^ z75hD12;OGDn3FfwqW0a3w3cydO@58v@>Io9~2u{eg3+ zdE()`HN!46lgYnUz@k3;BDpbsd3*^%>;7nr6#K(c`B#STDlG2DX9y|ma*3G*7t|qmf5^5 z18);9skLL=Z<+~^w4}Q>ygl%~Lwni0EeCH`zfps?f1FRf=Db?Fr%>K@ z+xDqvZQfRZx2>b5x$o&e4sXq{)6_i_%G(ciz4|Six0T@S!CPza7V8t8dXgc1CVg3& z@H+=m7c#euwdOr-)9%7w+q|s;Z!5M%;ElfzLY>iD^b_%|8<#g$r}mwI?^^n)S(l{6 z-8HL}wc%}#O>L&zysZXrGsaGL-%~43bT)YJQM0ZTrzb>LCZD=WNQbf3y1Yrdmg34z z3~xifa;o$H_pSkN{qAf4Z<_uC_g?oilJt{aw|bEiaCy^I&*3q=9reA-?yTN#8+Y6F z*rVGVcg&RD>w3&OqCAzi-id%)YBXZyXP)NAEysb^#N5S=bD$2XSk7Z`TX4$P7FTiG zrNZ0sRo*s^zoGc&G-O|~*8U^&7Hfu99>d$=yMGJ6eY%~u6RNzeyn82aZL`^nbpu$Z zQ}ZrL;c@S`6jjPO4R6awzZ-uY{&wC@tnxOYl2cmi^2T$KR!MR0&eDdrQ+_}HXy-j`Tjg!|QK-pwaA#{&L*#HsTLM);heMQsu4V^qjo0mZVd#C)M(e zXY=Npji*5EnKtLsyyXA^m`QFnaCz+y_o;T?nQ*_0gvFaAe+kf44^miTJ zPOI`ZW9ClY>iqf}8EB0-xvAShWYE6FnskxSM zNz-2pZ{=`bkIS3Xi4kcCtmXvaCr;qHlw=X?d_{4k9O|2v#Y!< z=ru?Gp7QqA^h`fB*D_8`54GfBT++-_IR!fTRmOd_Zyw3dxUXi1S)>!=%bH7<9(y}A&TJpr*DXH_#%nmcw8s5JCg~MKV zcssYs+n6VF^2Q#hGx}+nHa(46n~yhg!d(=prHM&#l9q4R+#Rb>mpz8Jm-jglf82Td z|BrNDmAAfq=eqAH^Y5rSd-Bb|RmkMoe>iWp=Xm%>C4d;!SGj?Qn+vi`ua)ZO$ z1y$Z2d%6y9thFQQe7rUD1SBTo@|H0H!`q*3pS;fD?ZPT=H};=b{Cg@*CSxxx_i8Lp zI^G(!JE?A=yj|SqVfXh>FRJo3WqBRmq+RgbdB;-IlUi%~WDnWAF&i~AXtcDJQ=-=7 zlsr-`J#QhMl&hQj?VPQ9;jg#c{`11cRoDBz=fB~6 z|8_~0w+B~6;4RklQZUIkf?CfTyJCh|yO#3As-@>mQ&IAi!P`TB|MT^i z;_q`<_fiu9CykzQ9zWxfrk`35oRqm4;@pwcPd0Fx3{qw*Zh-5!_TTrf0$1U!J$Rcq zbV2rmDgn(}l1GYjt*>TIYTrEJyh+ui6QrM7>YN^G$VEUoY&k2Z5Ym&25dCh5c)ithb&N*X1k(@I~&Kb!$=NttwAP9m41;hXn1kvBqJHB_G zw|nZ`>N(-g_3IzcT3b_HRo(qL-7^E+JI3a%3h}nN&@JeT-*E6kZP zb-WG{+J?95J4~!-^H!C3TRf*2-uR4V+qX|1%eItST8tvc~Gy6f0v?o;2n#2p`=H*2+v-Z@0oc~tr1;jP?`+qT-g z)gazFe)tD(IsQSQwUUvr6cs&~E0uRz^MAcg`Z&LL<{D!v|*WYoP&08JfZ9v8G z?)fHtF3zKeNciM_xCi#L%`uYgo^P5A%mByx+EP~y1elo`F0y-tZyyf`0%_*6|Gqt-bURw{CJzUdc<3;&+_Dr znP*$-jbqs6c*)N(k}Zvu{4Q@&q*O5uztn@TpL=t8vp6aCJW*CRyd8M^nmcXY>Jx9j zRGHx3r`)Z^mip0*EvcByF2vC}=VV(m(>UT?-ZY)$5o35e;?CL!+PpO&-q!YufVVt) zEB2YyihHcfTfj=@eY(bx(|2W} zEt_VVm=?o!JHUNTW1U;cduEH*d0eYfilN zu0J{aeJU}}J!@h(-q-fcpVnJ;_Zi3dGZ$3Drx4ZiSg|x zm^a^Sg&eQvE#xc=Z#N(Lx2J91S`lxbG@26rJ`LGttY@Iq+aJ8?S3vo^8{RG$S?NDE zZ>@>9(PQ%DP1!@SEO8#TB~E<(QcnG9`&N|VrB0;ysQl5MZvnF~yzRcJLK~a6HpE-I zHF@$DW;LYGLG1Unc^!DH)ThfEkCJRDnq~~eabo6<{sGQ7`8bkZd+O_uIuuq8O0f*vU!!*mh$`d8O$43(NyPn%Qn1q zKDZtI0{hILXSE~Vs!b?{H!Ek+{a9Z3QQh1#+fp9(b3Er~KieF`Hs_S=%r0;?DN^!> z8Ou4jbGA7@+dNid=f@jYl)AF28{WEYd4+y~R))9s#9NJT^W@E{Rm7p>_kAj*JiHE4 z4Ev>c_k7cg;uyB2D(v@N0UpaX*VWj;yh)iQe?UdU+rVe)JYwIc9f-G0EvDsnpK?EJ z2lP|KB*(L@85>nhFmF;7?p9+qNZG{lSAbX?BvI z`_aU3yv7cbvt)S-)4674I3_;ctXUY|?ycPE37fah#M^*2(}V9*?)>*;=f|71!bK;1 zA*_tlTYS9ntWn+Y_UhwLm$iB8LcBGb{s(Ub`uCA|H^bP;@P-ZzZ#!)Hhx6z4U5U3} zfBc>E#!o%l!RH%ChS}@5$7=cv&zq*oZ@g7K{h~AN^Q{~4_GSAS?tQAvU+itdpOWHU zdEflOy!mF6W*XH>`u4-|;d$e10k%2c&K!K%5jJn#iMQ#qa_|Ebju_93$p{GY-V!5k zRzJnEs+?D`c{Fu1yO3l>O;>TmxV*(QYs1@j4=;2+FT78@b?iLTy-(wqwUjfaxfjc% zd%l%aC(_D9<#&0LGKX9N!&|#Y?_Fe{Z#{{(YIAe&CY`I?rzVEuqp~%1HS3ny1u2{4 z=UH$J+nVZ{JRGC3xd)Ed^Twkzwl!13+opftJl^K57xA`nTMphdwMu)8d%kIAWUX+q zr>G`pm>8)a_Q%JY)w$vA^6h8RFR;)2d3|r`%=tEEw{Goi-ue=6U4Jcx zH}3rRWar0Qz-lLUGV?BI`iYh|u5Q_ew^QdH*23nkAMsYZ=j=c3)BM=8R=Y%a)AW!i zZ}nb$-TD1l{fV~?pZvjF3IF-2=8V#CCB-Dl+s>5^9d4g*1BkbUmF5JWZ%MwVifZM# zXUTTYG>(+^PV)0uwxxJqKj&mylP5pke5)Yd+^4I@rJUc_HjsFm)u$NVct#QrlAp(h zvE#_^@>XaaYWmTvOn$uOQQh$N;t9{cZl7<1h_~JgJ$TEbhoW~izR!zhB{ltU4ELbt zjXU8CY;z1(lx!X=#rWow@`UG2(=DnS-tPONqx16y2NQ2it9=lBpGvcc%I~|XzVT9@ znO%^wNq#9?N%^Hd*&m)aOw1FJa!4b16C}ql(v`Z&6h&sXU4D z)_V41*W0`eCEk`U{)4v?{(LHN$Y+!kqwy!o+wuENI>F{`81Xix=G>CKPo)*ptag}~ z%r0m$NF8eYzWIIQH9Z9L7GXtmpSE58pBrr6h7)gH2Nx)Bnw1O_li7tZ8Kkih?qDSE zB7%97W|4>A@b*fjciP&#jUe9YF8hPG0=-W)JFa0nQQpq}snaBzw~@r#hC1_t?^Df; zq?HLXR?4aIYw8A{Z<-96P6{C=KHjWVGUwZg9~|QRzTHv8+b6?fher1KHkx=FzM>f3telZBz^f|RzI8c|W^Cc`rs-CS z(abtg-mYmr`xKkEF~nP|1|J6Br~KVA+q?qoXPaY6%9i@k^QPI;sAebClKh&w;d$dV zWZSB4&bKSpbX{rlHkNp+KB^ertelaYZ%O8qc1q7%lHJBpD?i?(bqnaw@HXk?8Le#I z#u0DpzKWQ)0RD?u!=UE68( zbep&F#M`pQ^MmhG?&0@j2lMv(n<-Nl)9egy-%dHPoz2?>;%(~KVt8ZD*_L|N#7JX( z{hH^F`(ZnnH;&YF#qpBOc{oP$Yht9ayfRE$#`ew3RRV0oTi>_;xzXlrBJnnQZ85w_ z`FTbhYXGr{S&2ta8qu*H0qeIyCzz{C&zhCD}2}vFJS^ z=hyV$^2Qk?n=^-rk;ca1cX^9x7KXQ1zUy?1eZEa5-YQItfVVhSDej#4@g{Yp*}*Vl z!{m2)({z$YMZ?>zkKXy1&D#{>ZTYtm@aEf_Z@jcVVf^v&RtU4>mC^IYqa-_Ec80g} z>Q{Qv=4~qRHl^jq!S|{3-css^HfR#ad>+&%iF|_dGf|@8`VD?ju*{Dr|F1-1)gK+sW{TnHt`z?DFqE4sTCoc^ll})BNt!Fsm2U zE^r>UHNCmKafYaDzOUoRsh=rR7gZ;m2b~z+PB^k>U5B@)v%EE#or5>cEVx#o*_!^7 z;Z0f@sYA&x<1SLqvg$a)^Ws3E9vqUQ+31Jq211G=J57Hmba?iKP#O3lt0C5)`a7Ewb?J_DSX~| z7Hms1lKjEEN$Vz6WIx-eZg`tN_&xgdGQWM_`(l>2_45meH)+M<@VmTm72XB*Cuv93 zLoja<&dBhVKKz|c&iVFImbXuTjes|4Wuo#&xF$vSY%p(eROFn}%5aQi`;JYNw~CkF zxx(S?EhqqU=y!~2fQS#>-_vvd#wN{y3&{UTyN`6hwBx79OVybR2Hfkwr2G>XB_cT zPVPCg3+$1)l8BY6uwRp(W7y_AlFjjK^IXF7#@VD!;5X-6&+=0$IlR4|6jrHUF$W;r842y>DiDtG~p9H)%$`eqKGcee;Bgch5Je z6UiT@dYG}?58G1CM0xvY^`3nRxXj-te=Ez|4>cFN_o>vm@*a=m9WvKqn|qV|93$B- zZyXsWa~yd%r?0K&jc3iavZA?9Z#=PHYlpXTS>Bcn_TWufHxk}_YiXWzjtS;1j_STS zql(w_7P6w@ZKuv-&T@ErJImXYWfAa}$Foo@lX2wa^^BG`&Meu!GxCj>>bksXGII>u zQWf|OZx5{;x7Ok9oh)yI>n;htPx(v;s8z&|rZ6U-a08&_eQyAALg-kO)avyQ`C`7Cceh8M#d*NWTbxl6S)G16G}=f@joC}f*g z$>lAo&bbP58r~|OaQ~SOZ|`P#tGm*JH;HXc|2&p$9&7cV^URdSavsU%{4Q^tnQbYX z#?KWs@lnNa-S~LpZc)+j_SF+NPaxiw{_o${SD>Hgu(`n(?uE+B%{CX%_%$(7PT#S- z#L`%fae?Db;LX)xg)Qzd8*FmD{mowKc}f_TH**(aSj*XFGX@z!cw4&Jzy#+Lfgj4i1cJ#SKnrDfK^yh*c= z{L*T`Z+Kg>?7uy1-l`IBzpRabH||06yh!mpmTiugY>sD}*Ny!yZ~UrMMe_4lwxxJq zzm!w*bN*o7IFi?j{gQ1}H@v+u;GD)bZ`Fvm@0%@i?^7v%Y4`Jby1bRt?6|8qdeHM0 zvby2zm#(YNwRx*fynQ?|0^T$$7FA5hH7R;;sppM5XFE(Mnmn2qsRtgLC~pUp9ejq( zTMgoE(06(A#xs&;UC4etZ*k8e?v+fGw|&lBSkdOKCh^v?)t9CBJ`J-|rHi-lGiB;x zdVz(s#-s%uek{pVlSby3feL8-K6Dwr0gRhI?b1W7rN;%Xh5HnB4-3im!`&e_)Vris_YXljLt$?SsEiN?VXC+e4S^)ZON-A@MfcYec+F>9W$jPoPLiTOmDBW`A8(q> zR*d28s7rS~+vcq?@z!fz#Jow*1^cZ&i(Ne)%eE#D$FrRuZ=8W`sS}QoY#z%o?3Zkg zXPe^_@y1kRWejh>G#Y%m&090# ztxCnO;yd50ehO+gGrN!^KP9yq@$qJ@p5g8Jhc_&>d23F*t?m^gZ&p9W@)q}waDJCJ ze$}^gDPGDc`K3Hz{E6~*&0TvxV)NF5c$>AL7~YaxnJ}x$V>Layym5v?wxxIM+%w0x zyz%<5%^7@cR5a(?>`B)Qw|Q$xybY}Kb?|)}rk^lldA^!Fni$TR*##cOHb+W+j?vh@ zd3@ug9$em}$dG=+Tf2+ycm95FE8?wTzhZdfCxvb9js0O(iSuxNjqN)&_ zu|Km5?D4J0@tlWkj^R$&<`^E!Hplqd!Mtg*aVG(`;cbVGZ&tN=YeT$kt-d<=KGpOi z#Yn3v#V6?xKHrk;B<@-+Z!uLgyfxYPN$=0k1kJZF;OyTp! zvy*I|kz@z+#(N}XQ2GsT$8W6hw#{2R;%(9wIe7E!!8e{)&v&eEJlEoN3+7Ezg=5&} zDr`%gXvT6LwmGL{yS#;|$knYphPU!v*MDjA)}DClS9^{AKIIv4-9oc9eFpPZXfq1a zkIP$_Y|>b)x#4Z=uHTlkdFw#Dy*IQN-YkZT{mxeECk}s@-bycTnpty9e!Ou8-aE;U z7{gny_fEUY=B*>~R(W|bym2l5djf0+^iag7CWdQ;vAGu8oKw%6l!5)c8eB#4a}4K^ z{2aqM+2*lq=f@jokZh|f!`lvP*BxT>)`@tlQh%-fKIOWaJB#BB&DQf)Xr24^6CZCD z1BSQS1NLgy_MLUy@z&gn&Q+;yVe%Ho3P;OZz&ab=o_zGDKilV9 z7vk;1RmJco?N=OrUTwDHsFm3Tt|HlCs!KT~KiA?I$k?oL5mT&MEaH#bkCtGm0~4Y~K}-dXW6wgQk|AH%&#ZZrO&n7rHMi zYxCBfc&opr7~XhBl5N$^$pgnodDzeKaoG8tZ`=uY%QnYIwr@P=@g3{(#u+p=S7+NA zYj|5S?E&Y{PkRt=)tj!1_C5{yj1h&cn<)%veot ziSm{{^tp>{-g*&lLt1>B-+da#D*5L0jpyBomN(8UbtUrK2ho176HHgA21x2@m*PI%)L z)~s7FZ<-8o#Q4ra(?c@6p%cT~^2JYnV)NFQcw5@~yZr7`%}fg|hCjb&c7Z){cPrIm zKigsYVQv%UZF-wCCfmIABi?3BD`DQsHh(R#&!oJAg?YYlhf&%2@y5g|J2B_m5o6j` zvU%%Iy!HRFgn3iWC2}SUhqpX-wrDk^vFvww<5zk8B|pam*oL>^XSI0C=4}A+*0}xm z!S^Zmz;;00qSZ(;#<#a*c=PQljumrxi=zs18s3(TJk0s?tbxSa=9$It7E^CInM`tb zlHo1M)st!k^A_Qn8{Q7yWmpaSd>cf(E#6!VZ&CFVRUTUVY>t;~9xKIgha4~Y zISE8A)?- zd6ObT`VDUnHh;qT^Q_^-+m9c5@FqQ3l3(gy6K_2`IX}-z`6WNs4d#s_rEJ`x6d#pe z>OsoS@h)%DD9O*+q!{=OZ?`XOa)EumjUe8>{>6hgUSaM*V}}{b`89bshHcIh%o|5) zY^jRm*W~n#k!rcT`DWI{qq^a(=C~W3pWi!@c$@nEhTr#n8oUddl`P~KX?EOmqP(4Q z?sGrc=i4aat=A_xc;hpZZJveX=NPu7c=r3+Qhv_EF>GgcL5lSCb7mgPHpfV|6wiLi z=A0bk^2V=9qc}!soAa&ztMy;Bc^gf<^{ez_GUuD-N#wbQu{E_ayAY;IX&LMCrdc0p zb^&7zZxzqlFwo|04Dr^uPeQy&{g;$K8Q%D0(9ASoH;etm#I@LVdE-|#T}d%kw}!Wq z8fC`2d0p-EZ5;8o ztm?*O?$fBADV{s$WSe6oJ3MdJYUg~)r0N{H_j$?%Q5KI@OI-tW!l-i zjVIpb^v}T?&y;PcZWw>u`K8`m-lR?>zm&Pq{Q2>g$1DtQ_w;!E6`QvS#9RNxIe5#X zRwPe%k~{12Ce1pkt~7Z#MzVeT4CaliNST#>!`mHmrtE0*Hj#L1P;*m!_bH#ZVQgu> zQcg{LW*0bvG|Jbn$;mNn`_|%k&ck*vZyd=sXI9$(JKruDHPhy867g1JaE!bu`;6pl z^sS{o-+VJi6(7~f6Uaaem(5Wa91HrFrrut%l@}a9)w{nc0QX(ruX4 zOO&^Lnop={pKnu$xA}E8$9JEGS*5tg#&^CI8pGjv;~BAS?U6a(UhCCxfz8`g;%(Tl z66THj*Svq=cx%2n&pjz;e!Ouuwl$q_j5L=0lFfP8mg4i{jk9r;0Ne0(&9@C-uz8zC zy!BX7!n_6apOc@c;`8Gzs_IGR$&a@@IyAhsFV|^?&D(V1tzm;L`ukL3UgPH&wl(wO zc*&OXNb!2!ID>B$j^{jVa}3*3jKFhDIEHO0 zUgPH&wu5=&NVd5v=8gT9ZFqZR>i+b{S-+icPi1-gd~BY)ac`y3c6pO#VXdmyhZx=| zDZj=arU#cdO~pK73~%308`j@B-=5C$Hg#hlgF~3ZQmFvUgPH&mp6Wu zJ7ilE!|`lOV?w~$yy30Q{B2tu-k!4b-nx2Ds;|$Cg+Z+>M8{V$3de^QFZ_j3V>ohS3Z_F*{m;4;VwiGY(5x!wT=zppLj@%2k}r98ay$?%3w3~#$%-Qhgve0x62 zTeDW%{u&d9!MH@lyz6li{s!X3EsX)Rp1w*0(2K<(zM? zWO=KzsTkh)lM(kR`8Cf4k7ZlRA5{$J*Yn2JH8xj?!{#11zvO4q+*wPbI5Y2QK)m5? z+TmqJI=sD_<*ic3?fUza>vBJ((RNqfcXqzk`)h|ADvcubJS>9I8 z_TVi{Z@yy#o})zT@%y~5#5nrT#iJ#Qf^8s1KSZ7lus`rke;yq@K4SeIY)_i4y}@@9kU{=plshQ#1zi?7WeA8yv0<}@ODe5-OqA(do#;h zrR@>$rrF;(V#2Ij;qVs6%5YAXH-0rj+wgYgB~!{dyuFp>?d$Hpy7#H?K8Adsne*J! zgEyYt@6`_GO`0|D zqZH%ok2|N!8&{NU-&r8uoNvcpdH635Z*OOL8(g76ntlOke!Jk?oAl&Ld3a5v81{31 zwmBwvrqU?xM2g`mQN{S?Aq-@+R+Z-QY8{QtS_3j{tw|BCt0sRzv zZ-&Vu)#a7vm|)&GGHyGnSxb5JylH02RV>@^R{Nq$YB{`>&+^t}L7u#EZ_=J}4BNi( zJT{EYb(7(ZD~9P@8mq}4%o}IcWaD_tHoUzv?e+N%Z|`P#+f=z?+PP1C>-xrXpVC;# zuNliRlAYNFO%*9aY4PiM3ppdh+qpL!vx<0I`oDi)Ux9v}!zX=n02i`w&WqDGXWS@6 zd%(rrt?%k_e4@Z@Z?yNuEN~g#DiUv_7J2Z-7dFpGvZZ**@0&-8*Z5uDl4M3xOIjt# zFXhzp=9}4CXLG%EVjEfd$H2RI4rJiH|p_VoCXfd5f?s!`qCRx1DJ7R)u)` zVM&a<@v|e@5zaT~Im~srR^jj_btw6H78*NI-j=E)n^joY@Kqnw{ZeS3(OH{a@U%!qs7{K35OsF1ed?Z+$X zePr`in|S-OVb!#ApGrJS{ct?nQoQ8nRg&^Ze$J_h3Fb|cfh%(7?2p6d-ZXVx-Z+EQ zA;+*CP|@&q`m9F|wRx*Uyv-Y(gEy&%-;+O>x2R?suse~Q(waV9-ZY~$om(-6w}Fk) zhuOT>hgDO-s%x=ZQiSvcJ9+WY85@L(%H-Y zF!`ezt7j}s=hg|3bCozh+dNbDb58biJloOo7Esaf)@$9|EjDlUiMQJ0Bj$}~TFADZ zw=lC{Hp1iyGd5b@0y;6gHMnE$-)!C*5O3?h`GdEF?$eTDFHzn)Km5+YHg64yx6hha zPdoQ%Nv%x4vr}{vMHQ3G`4*%81 zUs?yrFZGi=Z<-D@G183K&v{(lI8tMCwt#rU+mC}+PPKV!M!fZDS2OM0r_y(N>}Ol* zK@-DcedBqoubn(^z8z{-LyBR4c;1wq8{UR48v9S1x8}rK+nG6dBXnikFP&J-h8VRLVPf9Qipl$uiV~(C<4-xn=B*v^wrzVcyh-`l z&*u-@9218frdH%ncd;jG94ny7>GI~gZpwJW+sBpaj<9)aPrNPeQ74_(eX8uCXcsu1 zZOw}L#%S{YU*^qsJvpA&mTkls-a7oX`iC}e9f-FvALq#%_uy+=eHJn4o5wev>n6h+ zSM=>xikEUqe)oLi%+e^1QQC&Lo1a{^-sY_%@z%a#-E>~(o2CcNTr@G7TD~#K@aEf9 z9Pz=tMOe}BcH;IO2im-KBHpU@j*+)8bBUY7{CJDIE8i94ek5DZ8)xwC1o4Kq0j)=$ zY4g^Zc>8KWF}#(h(@$ededh6bC+9t@$)o8rvkM#v`iRjx(^$G1e>$Z;j4+u)2M|btT@G^vl7Um8Xa+t}Df`UlY$U(p)%( zZLTHRQha{AaTU(YwiLsD$>uz4bG+o|7`B6X<29EuOMb2b+wk^rg^g$1ymceq=6~+N zo0Y$a!$S7+dUBt7-gs1Lw1atza7Ko=J?lPM#^$X%@iwAH{j_tRNC;6p3CFSQn zUEcUrw!?HARh}?CB*PnKYIqw{{!ZuLpY|Z$S`PByEzGQaSKc??w>OtJ-wdTCJ{jJy z8iuzYhhILxKHuIa-YP6jh_^7iqhU`I!#(JE(`44faP=hZM0uMt?cIZH-g**m%WF4C ztMAjOR;dtnN6%YSv)1H^YGpW2qP*SHV$RhzZ@q}OsYCPRjc24;$wG@MOx~nb3bWdY z@-}^amlJH>dJ}KGm*?P(KmTfMUL`4BiebOT=KNAT$FR-udfqq#+nkO4Y;z3T9HX(N zvFvA?bGp3ot31lr<~*>?`?nbtpIu_})`xiORli|6@%vOWuP{%vCcmCHO|~#G$;_0g z^X*pIq2cZDGd`|u^VXMmYdF$_H)Y*Y@x5F!yv4m@n*2;!tg|+}T~zNF=ifW`Bi=Tx z^5Bi13qDKPuUP?(FDX0LmCyT$lU7x;O48Uwd0V*GDCgfh_b1+#HfofP^gfkltywXN zRmsmeee3FZlQK(PNq+7&NjpB?teKkgt>>oe{%hZ-1BkcDVUV>7!T zy(;+&ZFZV|^5ZR!75LwMy8Euq&qo?aymebs3V743s&9<`eDlp*Xz}^+md7j%Z+jhl zX&w7~8$`U-Z`wGW*L|7?|3&9gn6Y`RLs4BG8!c~|8EIm;!!S12a(Uxd!&I@x8s5%$ z=ke2Q-UbtIKa9_lH>+08J>?kbU695v#c+PMIbO1ZdE*_CGH}I`vbi^@2bVV~lKoO< z$&ayyw+}8l`Uab~A;jB?bqVptbBV)dLNxhXMC$hPT~DzOc^bZ8-7P>xT$J;$V{;z1c`W<67TX-}@+L*{C`}A^g0bd&+kdZj z#@M`#BHmX1SPXBPy4)w*(pbsQW8=16-ZWiFG1kf#eLEz@`_4|vFXiF7E^j=FBiZH{ zrEPdC_sDZiY~Dr_Z}ZzXP3Lu=Dtjnec`4p^C8a#RewR1j45cMLm^aNVlzU`&`?6l6 znKo}@h_~UhN|-mzeB+48?1FM7i+yV1=unfB`)8ZS>Uk?mCl6;}TdIN>!`qezemT$P zZ7lKDa!W$Iai47ST#~f^INu^)0mECTvrl-+=4~AD_G_nRX`(v69wdGwf0A5DwK6mH z?a((~s?K@Xmg2+sGc)D?Syp1Q`mV6KaZ&uxcy?waP{GKe% zP+DvzE!J5Z-fnpO*=udyCJ=8^wne~O9?wFFGa2j3m-y`XSxA((!9%ufuz8zEy!GnV zJRSNz<=L^V`FzbWZ2RW%jSuEcdV+lY+?8bWtR*R=hdi zRz30Bjy7+Th_^QLW8}^1r$nB5-W~2Im^Y40(w4dn(}T-fm}~)K4R2RJcK(YtZ(3{(yDJ$$4S&#Sb|DWq5lc%iC8~Tcw@r)XG!D3D;sdQ> z%<|T}LECiTeaib~ou5TKuwSw@D;8!f=hx&3=1r4b0{ji^VV1F=g94Tes7`8bc zwmIKYHRjQe5BcqUdnwCX!%@ZX#yPnb+foes!`NYJNn?X~<8Gx4>}Q+DvaN~Xcz)8^ z<`}l6{Q2>w$!x_K-q!EhX|8j=y`1H(%GVyeasJY7>v_}6RKtTN56486GnhB77}7Sp zjh9GRZkG^X8k4GskWF_U7{Dn?W-p1GPYvvVIOfYXzRVkdAGIcR^WqA8~#7*w|x3XE@-fPxA?cAp^^^^0QiE3wk^CZKY zZ&zXBrCnpvVx6_&t^J0kXE^8E>sj8mPKbavX{KRTPct@5-TYQQOhwI%q!_NtwiM6) zVBUPQac0Xlygl^j@>&jWZ)AB}_-zi}tUQUGOHs{TlP9wan#eG#5qEw~Z^682GAL(Z zcssky+iy6$y_w~0RLc%&=RTEsP}YrvPp-wb)SItAzw^zvO5E|9l?l(Aau$ZSN#DOl zzi#)p?|a|M^44xjf%2xDW8Q2B^TzwgcHCXXkv~!1%A9c@{W|gAcq^CXt@3&g-lAF! z>6>s(e$7}-jGi~n#_X|Q)43EQBdx^x!RIKY72`ileT}TO3uQ%BkT>iV4pfXJ(sc2fyKMTemXqzdwC9%iF9DozjWl zr`%^r*`+|5cO=mRZq>DSJZw)BR>S|)7y7BR5t*trVrtR}pbDOuy#M_Fw z5%4C>)Yl(oReft^c0p576XRPwOuTPD@$qKO!tgf#+U}Rzyj3CIrv01{Z@#O<>CGl}Yw5 zqr<3jCd%7>&99ke^Hz;`Yw}SJ-gu@G1MHV=lsFENf|Wj&sAWX`}CQ0Pwi*(R-JgOS)ps%Io~*s)DQdF){NyC$>y9ITgoH( zUEVm7E3#i>^H{buV>L0HlWiW${$SpC6x*DE{cLj#Y{T0w2ik-mZQg1TZ=Y7~mJYm6t(rqa;5+p==}G@OJbI%SYS1)h6CX^(}@sD`!FR7N)m9cym`E%zFOD+xtg! zzscsU4)NAuQ4Zet{A8P%m;4+f**unQj%PpH9K$xpXl!@2eX~jN?3Zk*Ln)s9(pbsw zo^Kq5Z)`WPgIieWexbvuNX)aOa*W}UjRtVi{ zW+cS~^CpeT!*6)I?5usRuz7orc>8W;3GkN3T)fUe&d<*vlcVR2vz3(1o%{9>A8)+2 z=*sZ6-%Z;dv3YAsynWKJXWF?>rP)b-X$5@ar96^f$}jmjPk7!mRk&hF+41pa&B*XJ z{=JIM-?wZ=ybT+jgE!8@t7`F_^UTr2aDI;0*gRIUxo0WH<&8&4ok-c(uNliR(pc_? zV|?vk-h8V_@tBd}t?l=_9&Mj*&55_Jt8?%s<Zu&$pJu+tzV0@>a+@5}vm_b|mLJkT7|8h1uqu;dzrXa~0Tz zw;!HA*SSwy5pRpXiIF#}pU77`O#bk^DR&{USxfzd=S{iJhPQuI?7iGR-&zxI6Pou< zm*{;et&-$d&b?@t`8@`Yb$JU@k;k&l6(v8%NVYUq@^hZ>yh&@wRbU(5E_`$HTQ+ZP zh__CYBIZrgn5ZX`UKo4>>uBYHXN3^}L0t64gw@kHh8;+2&f3pJUkOcs*~t{+vx|oAa&3y_X$e^VXhtTQxNY zZ_50!o?V(A!mN+WTbOJmHCD4qdfqhIIL4Z_;q90KZI85h>p;9s+mI)3`~S z#qh>6<+?m0jm={@r!*nSgz&rCS?feH@vM_e!27W zRJ#yw6*~1xM|z)1v*s0$>@Z`coRZ(=O^Ph-ewVkHW@LD)Kez35_W9P8cw0FqV%{`! zNs=q6pJaF|oS8Cpnyxf4*4i50R@O;ZuzBl7yiMC$0=!v$#`@WTYw;Q;!y8vGDVs@) zcGkR}n4RJ6pH&a~tIb~X(~??MO1IfP{(>N z*}U~2-Wq+FgSR|tMSe!{dsoTU?6NeMVBR#DeY=u6;XInUzA^dn=352vhPMl=&!Jzq zoB4eo?-Os^x98w3k6MxQ7N!SjXE~>yH%&Iq9LA2LR-(Mke|q1~?encC@wTkTfVBEP zjbk;U%E_k$+tTi5c7Y=`op8LAxzPNH^7iYB{?5(4jN zRvPW_yz#mPtbn;sk2`nc1p9pJO}q`MI4~XheAD!!i3ylX&ih)EQ#!RWyTGqgPFl)6^>MG5MWu{3Nn1tp>-iEyc&-59ZCcV!$j6 zZ-cfj-_z!8Ao13t#^7}5`!t}361itJYnUHzn#^&;gjwNec?+1G;q9b7tG;dXHi&qu zHz)$$0($s^w-R50M0uNb?9Xd#-UbtIo4!bhw=hpA^UQhJ<`|6~?fJ%?u&wDt6XTnc z;lOi?Mr5Ino z)B|_!Yv;$C)VWk0@rJi|noPLJKHr8CZzbwNt@^5@)o8O&g`2f?y>pt z<~s|-8{Qs&?Spr1-i8rxU6<$JjoD^f>OopD9?Q0rhhx~5@=JL%ewR0{sL8?2pQhJHOPY%bOHg+WqpI~m^M zo`oiVe7srvXwJ78O`mRW^EQrn`(}KMylGa0W32v*&LWP9%8rk>Bxj*njc9obnBD*0 zzjgTfa+|mD#M|6;#qh@O3E7rr$}t*S$`ddzFAli}u9evZ=~a%DY^g)e8I>*dpz#Ou z#vMx8IEHPGhu`pa-rlvV+PqC5-ln%0k&f*?<$emyj_-Uc%$YKEF?DEoJL8P~x7)l; zB;NW@E@iw)r;6_y>UonYN`Bu?eB(8BIfhBo&)T;u#2en)_1Jcl&D$j6t;r7&@WwyU zozGjgIfmyV`8h^ob1k-cEZ54cJbN^wIG!_yvAHhWnw%WZHjj0AlOlO`Y;!iYIS=9u zZx?^_tn>4@CKGS1+l)**_bEReoIegb$y(gM`=pmtC)`yWw#!=_Rgly0Hu9rm|7@Rc zQ;4?;({u2~+;VU1XPaZ#*2M5GYvQFC_H$0DAD1_d)Yx26V{3A94CiD&+dNZ_XFHfT zj%1swuwSwfZ+QEn?BfnmejqiyS-V)Hhgc$+Zm58j?G z9QOF+VLJ>V$?%2_4R610?n(b1@Qa;xc=VCS9)F}?qnY)G-iz-|yKTvV8(-0yiE83G zMq_L0N->x?86;a1?;8^)p2zBO<54_Q_Oorp7~IbP*WEv6kGE8YwOtiG!`PaBT;4PlIWyauv2ny`dT@EujLM^1!`pe6mznGE_GFf~YIAe& z#tgHaM-N_q^b{t)Z=ZVJI5XRtZl!o>$EEl%wOrn$Y+RjfO*~=@Z)Gn1+dd9&Pi1-A zvMmR1Qhr~*w5q;&I6kVJdfvFYudV4;6XRQ#w-&pOrN?YS&(^?Hs?hrUmJIpL>|$FeQ$Adi(|H2%ykgsCWX!hTJD zO}AlUqUDWeXLV)n(+4VCG}htm`7CdnK8b)gt8T2HNu%1q!av`lohefnX1@ZuGQ4#; zf9nc|w->U!Evhswo!9xsb=eLxmn6sL$6J`0@>sUxo;8z}->j7#8s7GQ_0eA(-d@b| zHl|Mw-uTR9o9nWlZ7GKRZ1Y&j=6JTl#AjBXGjL|MH8D~=k7ZkmXTM}~9?9mhE^nGq zoIztFr{S&I;&10UyuFm=ZS=w%ylL_ma*XEm$nQzlWG<Adb!-`=D-vY&0={J!z|@#b44s`xM~33k4sWkz zd0SF*LOPNAG~`^oc0^Ol^YzK5=c&j$K<7*CYZ)ADvQg>py z;O|puj$!=4=bLY4Y1e%7#1ZfErm2$0j0|rd-g|XNhqpJgywx6_5N~<(UxN3kW=G

    ZQY85c$4~Pzcd$)VVlo+wmF7vj>+$Q({#vH zxH{XMhvV7L@lp)?IiBtOc#}He%v>F|;qCYX);{F$RxZoif(Db)>ibmcLGp8Ng=Poy zCe29l`_3-OJbKra3C_cCv1c^fv$gE!w*^^Lclm!kVraAm(gqHOj;&G1%{cpKJiN;>p~%5Ol~mUe_= zG`5t-*Uz=wN1b1lGII>u9PjIw#!7j7<6Yi3vt)A>$>y<$F$dh*3x26>^Hzy?8#FNj z-lTpC+5gA+=00&D_sj71_TH~|uz9OYytVx9|1@tQS2*vFI!WH+?)etetPO83om6hA z&07`XtzxUG>CpFSO#SD*n^Hf#%aK3bk#{T2!q=Y+Z&+Ky+mUKxw(YIDq@yYOpZVhjbjy~%Qo40Dj+wAo@cq@&o%DbcKKbiAQ>O}JMETXa# zT*mlZ@#mRDxN!(a&nAh^H`TR9>tN8pJOB&V-0WD z9dpo3o41<8Tg6Sq@WxM#RI4=l^}I#30#W7R)#jXRbBxOyzbcL57^Q7^YoEHZg3Vhk z;%#|{>1pRa<@^!$lh|3u{Y!Rc7k;m^<5|1B@vPbAQF+*gx9t}Uy1?eGHt{xPb}_u= zQ8y=7VdfQPtki$W@)l-BJeF;qDf?aCq)7H_Y_5WM!&{pLi~88S)gj)x{FD%Hn*NiF z$?QUs72~dD?S9cyAIw{XT^Zi~@zb;cHg9!_x7uB1q@DXz>O8{UidI9)sp&xz!|#DP zKC=t_s?PXu^K0xldI&z>xFXw{ZZ$E! zIep`~2R(0Ls-U7d-wr&a;(0c2^@+DRKNrIrp8y)0XDY>W4BMJ`j^VMAEycU%n-s~J zIYzR>>>B6P)a4kqIgiU5zbcKA`~hPPZ`-a+pK?yNeS6TvbAGm^ zvF!J?Ie+l^CS~Rf9K(Lemg3nj*}gfMd6zeSRjMfY13EFhUHFd?r`fzUB;KZcl!G^} z#l3MYj*90#*v&svN@h@DL!sL_Yf^_Qs<~>cspyuk<%udr%Fq7%&Nt3bXtuO^E^lEvm#QGA;jQTv=_hU88WV4= zdqu!oKo6084h^$D(pWuj((EL^R8bSJiIMV4{zQ43UuVIsHg8Rcx1T@FleailhF8ot zzi)gnZ@$^M^Fp(ub`Rek%(>_SOBHBw#4&$H9SaD1Y?z0&Rj=kJF!Bi?#0Duy@y zq{Fk2Y~Og!qZzANZA~8i`KGDJv+%Vw-TLO_c$YVxQH0gaefq&6gPYm+X>;POY4thj zMDA0bBdVmK$;Jl5q+ie!Htw&88;RW+}%d22zuZ64sk8_$Am z?oIQ&NMkvVWOIHg564J(!t*9omnv$yieqdrZ&F2#pJxQy@YZ+VE@_*$mc-k#B_6zK zdhp!^P0l!CGP}UtvK>dosB-%D86R)fj0|rTkMI1e&08ztZBnfd(s|vdR;@@rwfU~3 zZ#?%L{(R$XVLIVFk{u?0qP#s;_3*_uZ>@>9ZbORUjn7PJHKK~qtSZNFUCA%?9G*9> zE>+RQNMqSA&5rZ4&0}5OI8w4XgVHwV+uHtzY_)l7L%cQrvKZc!wIVs+q*_t^a&gX| z*#+K5ewA%0#@Ej|gsm$#Uz8{Q_BKkap!x3R5Jnu?m3IA*HJ8O$4Z&NgS3R)%9>8{Vp}=`habtsU{Ucz6!pI6vD~ey=kvjy%3O zIo{=sXUCCjbBwPYCXYmdriWnOI0JXcHpj5dF|ZA9$4_`}SDUx?#M``;1;m?XRe6p) zceXWoGP@vUlQKjV&)sTlsjg;hFmIfhZO*LmBgXLd$b;Y2v3cu2yiI60FRi{$xrZce zY4;_+rnlhpO_L$2m@u=@^bpLOCWA7@@b=eB*8Iultt0W)ZFC9n7S#%e$rH?5n9RI8 zoQG|WiQ7(;x8tX*y2a+L6Y*Aab)LK>xq5Nb((@Ka^|%@1{Q2?bJ8Q(7^X-_e>)PAA zbtc}{H2E-{*L~{S53fMnwx+jW-s0{sj{MTBli>|BGQ9O0cULu=w=Trn>~W=xH(ph? zeRojLn{O50c;C)_zr?0iMPJrc<{z6$+qUC*HPCEQU9!pVIEnkGDK_p~Roc{CLY_e+_RdzCUiI z&07!RZTWY<6W;jGST$7S$D8jiXgcxD=^Gy(Z`SM#ZP!|`nU#`9P` zZ(M~lur0-C{F)f901=r({d>lH%j= zXLdpAMDlYdni!5RDLXuG%FYdM^BX?+sLfj+;_dqxCBU1q{{ngc7FDm8s0AW{;-d1-UbqHUv>F3t-epAT1jb_ee;LOQ@YMK-<9F` zWOzeG!`r9(op^-J+aTg?*1QtnjaMTo+m}5(Z=9L!s5*~3Pqe%RtbpO|$Vru*-$yx^ zccZ% zHSDnv-){lUD&U0uqr*;#3<d9Ii*?} zzk9xEMoBSA&WLMqKaxK_-mHp-w=>`Trnk-8NaC$^m4#^{A$S*#PlLKd(OJ zNSn8D#M`7VN`SW{SDV>pJHM5eI+y%hQDaMg2UyB4`MFl2yzMyaA?NSEjVInZ*Zw^H zfA~J-{bHNQyjy#6wiK+mwJ%m*&i)$+!a@e z%9ip=ey*DzZ=9KJUURk)V|csjPn(w7yiF$F#*Qe4H|{(Rn|o$kswKs+pKU3Ro;OW4 zsfy%}BPZA5{Q2?5*(95*;FFHw?ctls^t5@KLcFzF6#;KtSL#jUk1CI*mYz3FwnC1H zk2h=9hPS8IRTGir$@Y!sJlr|k91|wL%UhUi+?8dU`*g&IM=vnEl`%in zp$zo@{q1sN?*Q!x?F8)%?E;zKMzb5VJG2M1C$tx|H?$A5FJ!*a+Yd57sr3NpK0b(lpXUoTmA-} z0-Xw-2AvL>e*ttRbQW|rWIlnP3!Mj@5B(jw0J;#m2)Y=$1iBQu4EhIjIdlbdC3F>Z zHDrD>@wL!((Dl#_(2dYPp_`zap?^WQK(|7-LH~wshwgyxgzkdwhVFs>1KkV#7rGC+ zA9?_K5PArD7IB=i*YH1rH)PLAiG=b;y%7onG+m!VgnSE1LS zve4_$8_=83TTnUZZRj1SJY@b>WCf@qR0*mKRe`EP)u8H74X7qm3#tv(f$Bo_p!!e) zs3Ft{Y78}j-h-M#&7kH`3#cX33Th3tf!adtp!QG)s3X(~>I`*(xElD_C+ykL(6tcuqYUO<2Kmb@$78hzbS4B2cECFCfcfk& z8~PEC+u_g^5cYLPtly64bH^ok1b2o`fxyB}*sqUZ09AvqZo7U2ZNuYzEOb5eB7}X~4ePPn zN<8v=L1#n%fxyM?SpVHWfHvVMI1<7d?C}hQ{oVuXy2nx+54%FAL3coJL)gbXXFxyT z*fdJnW5i-}@UJF$X{wLJvVTAnfx#pFrDj^c)Y} z2)zuofW|>z;W*j{Iv4scR2h07ng?ycG4*HY8t6Hw2?P%I`w~aj?$8;~T~K+b3p5Mb zfFmspT@F17)rY{-0gG|W?F9V|x(#|0Y7b3?zQyr(5Ogv02viH|4=sRx#j$t-bQAO{ z)C!sat;SLLC+K|WeyA$c3z`pY#SwZ8bRF~p)D#*Gt-!ImCv+Bc4^#o_2F-ys;bPjYcR0-+<&4o7ODmV(d8hRFL z42^)6;dK)YXbQ9rSImLX zMbN`gO{gFADfA1jp1(lWTR8&CqL5YiJ_023O^N(BGj4plVQW=p$$wuFzwl>!BB+X3!XDC9c)I zptGU>Koz0x&OzB{ zMYyhafKG;PfnJB&LX)9yaE%`TT?jn{)qwg!pFrDjy&n(V2)zuofW|>z;mY3!Iv4sc zR2h07ng?ycTY*1A*FeufO`ws`mw0=yJ9Gwg7gQeV0?mRp;B7-1x*U2Est*l;7UQkN zPSD?=+n_h0_Rv)5TfD_M2)Y=01gZt~hZaD;;;qLC&`r>*P%CHxv>Is2(&J`W$c9c7*;4-3q+{wS%TW>+m-2KE0{s(u1!@V6hrY&JzI~zdp!=XIP*3PX=qJ1tJQ}(ddLDWY8U-!K+rvGeGoibo zccHG(Z0JY4Z9E*h0(uH+01btf;H~7&&?(Trp|_w8&@|{fyu~~ix&(R@stpZ*KFhrA zy!}M{_h#rds5LYZT7$Qv`$2z)9)PMry`hhwZFrk{EOb5eBGe2T1Fghc*S(;#q5nV? zq3+NJ&?da4JrcSKdIo9)4TqNEZSJnnY0w?e+fXNH2J{2o{vHZl20aedg$6;3@RoQ7 z=w#>?=yj+qG#UB^Zg}2rFK<7gLg(^eu zL-U|5c)R^)=o;ubs0lO@`Vw!!cZbe^?t;ogU7%Ue2D~*-LzhEOLiM2`&|0@dfL5nckNpWcAG#l^3iX2K zLt9g+$B%)ogI<7|LZhJ-_*k$fbQW|EQ~~M+&4D)JqRO4uk#yJpt8&21B3YqtK4fU!hx}H=uUV6lfhjA{_`_1U(GZg!)0BLcidn z(_f%}La#tAq4Chy_&BvMbRKjcR0Zk@eF*)8k6A}U*Fw)j??I!W<@o5e2XrQMH}o#l z6`BqGh>v84LsvjgK@Fgx&=P!1+Zj3q`Zx3z)B&0XeTR>42Sb-Yk3zMf0nlfek96fu z#D8ywUV~ag6QMQusJ9>Vcjy7A8q^#52-=2^fX71DLoY(jpfS)&d@S4xIve^AR1xY9 zeE@C3$HgO|tDt9~M$m9*DLz8(3Y`Ys0lf`%f@VNJ;G^ZC&}Go$P+e#cvBh+Cr0|Z}2hn0O&&KA*cq_7y1O+j*q9uLpMS%LoJ|j&{z1#x({?N^k1kl^gc8X z+JcX=e}=Aso`ae|BcU(x@pgCU4CpSXJk$l61#Q5`-ZXSM^dwXt8UiiGN8z2Izd^S_ zZ$j;%snEChhUzZK9U~+T?su6HH3yiU*Kcv>9I)90gqsJqtC4MnKE(6~b=N>Cm0fJ5Xn6 zCbS-3EgS~@19}3g2MvZk$JY%zLVtyBh2DVLK~tc0_!{Cs=pyK0s3z18`V{&FUr+o6 z`X}@X)DjvGeT~ok`$FeI_d!*lp3sNTPxvb1Xy{t#dFVZ86to;)Z|nh`3Ed673w4EN zLqFnckHeuWpr@b)&`@Xzz6#kHItBVS^cK_sng)G`uSgDtE`c6}YC{8{&oW<|bUYFN zy%~B9Y7I?<*5K=u{h+@?4?xwR-q1(THhj%;EOb5eBGe2T1FghYFMB~}L;ry)LfxSc zpiTHn=1Ax&=ozRHG#pxruW5FLPJ`}%-iA6sGoT;v_06HsWzgeLU1$)r2w&;!0G$lo z0=*8kg(gGa;H#bkpbMdgpc+tL=o4r=zIr(xx)FLAY5|RdzQWf+`#|SH|Ai_;??dyT zE%>_V&(Jl{b5Ij#B=jY|LfRcV1G)<;4|RcNK^ySZQX0A(dJ?J+4S^Qp>!zKczd^S_ zZ$j;%snECh8tNeEV(1a57StbF0R4)ur%r%wf?kDMK@*_W_{!=}(D~5)P*tcGG#}cE zudeBY!~QM=ys?a z)DfBveUGom4uLL(9)s#Y1EGcZ+N=z867(;qEYt>?1g*u_Y5PMLKo3IIp+3;Z(9igq z?KtQL=q0E*G!|Nguio~C&VlZQDnUJs2(&J`W#>B?Fju9x)pi@Y6ne$*5RwZ1EGtchoPEKKj>5F7kmZy z7wDhRD^N>lJoGib7Tgy)54sPk0`-JGgnq);g-1izLeE3*L8GAM_zH0k=uGHt=v}BQ zG#mO6Uo9RET>(7>HGqafOYn8$&d@2)zoECF4$w5{JA4g!Fmws@C{!C70DXq9CR5Of z(9O_mP-|!+v?i6Bu^;qz=mDr2)EoK;+LlVqJQlhhdJ$>{je%CCQnU7g&W8R2RfM`j zA3&Q@so6(DS3%D}jiBMs(o|~BuFz@F9njlQCuj!rLn`&bq0nW}<4|2_5VR*P#0(xv>}!HBn@2-Jqgu^hCqu`sZV!; z{s!F!y$Q94rb6GQQVR}(E`}a~YC-*>1<QrjspP=)h`=P2( zFK9lrHI-U)40Ijz0@M^54XsF}KHn2M3%UoY0Cj`rKpRu3#YaF_LQg{tp<&P$snn8P zpi`mSp>j}1Xgc(LD)q%7(5290P#tI>v@n%gS_V1^`WI9dY6DGz)}~U+_J=Nj9)zkx zeV~t_pHrzXkArT2UV@rKW1&^4)bhQdbD(>nN>C4IF0?t7T5%L~HS{dh7#aaBOQlxs z2AvMw3B3b#hGs(RQ>j&lLH~fBfa*bmq0dvPuXcp~3f&650kwmsKpr@b)&`@YeD)sHo&?(Trp|_w8&@||~RO-8fp-Z4g zq1w;@=(AJ`+n4(OMCfMdHK;W-5n7W<{jeYOcjy7A8q^#52-=oPtv?pJ9(oaK291GM zrcxXBg3gBi1672&Lmxn!QmG%0gsy^~ff_->p{1$R#$BP)pgW+qp-#{Y=!aBl)1lC1 z(Bn{DXb`k0mD;=mbTV`c^g7fQnhbrDN^Lm+x)6E@ssZ(dK7qF5k6ez2ZiHTjT0rBV zuTrV4`#|SH|Ai_;??dyTEveMDKSS3*&p}O~kkBeg8l~G2E7Tjho(Z`rc%Ei1YHb00@Z^0Lkpl^Q)Teynq~0khh^~R zb7k=7U}f;~sWlCG<4Z5E=%3kt(zEF3_pa?NB+WBQzcQK2>IyL!e8c$DlgUKxkpA z%&ujilc0Y=WuZ3EBxr4_%x?Qb7eEg})uBGn$I#EIGP@rK-2lA=HHXGRt5Rk5*c&F=n^R@>JPNuRdKPL7jewS=%IvipbUJh=^bXYd|1oe+?Rw}?6h>$8)wXS0 zyXLNK+qP}nwr$(CZCj`9c`wE#0k8Yi4g<&&;+Bg2{&b83Mo+zT`>oT@k%CE zWaL7948vMn!Ec$^Nl+3UFdh5wR3=V1WJN6u#0s3pH<`HcQ53B)2|MvXCSFKnKvnd? zVw}Q9nfS3#0L?HKTX0(@fkkRmKzGc;QM{2!7!`TY5F@Z2*YHOsQ8JW9C(OhFJeNrv z0ohRpgRu%1@k1s_LKH_^Ou=qEl1UmGnNS`5uoP$TSteN=6haG($2Q!RN$!yrmCzFl za2)SsQbb2yG{z`w#0~tDNtps=(FL<{2rp$)MM6&0!%(cjW&Dy!ofyGrk7?M8Co*Zm zA`5C_0G8t%zRIMHhazZ&iP(YrGU-AfJ*uEL7U3j5$fS>n{Ah|X*o<2;84OaPJi1{n zj^MRS#wf^*1{jWYxQhQ|G9^VRbi@qo$1|DC;gJosF$gPh0pDe^BtS8=!DQ^hLz%3h zkP+3;7fWy&pJcMdMnN>kIBdlond}Z}P!T;aAII=kCPy>`p%F%61Fqw*OwQyegU*R~9>;4*&6R7i|qw8u2;#S@u|VUYzjF#yYP4qs&|#X}La!bI%A zeVNK3kRDag8;fufA7rY;M1C~I7;MHZnW_esfdSEgQaltE|A!a=-{sUHzJP!~h68kg`>ra>Z< zKs!vu9z2$57zUY91O2fKXYoa*QCt*8OH9Ca+>>b>Ksr=LFD%3fyq9Sb1NqPdqp=A$ zWts{pQ4U=(2Z!-WrdedITc(XgYE(dX%)?QC~U+H{FCXL0%g$!vvCM7Wx7Q|PSnFttifgc zlIflp!Dx?Z*o!AJJ;EXjYGMGE;~c)q^o)lhXoZQ`f%`JOLLfb=pf?ubBtFRWj*0we ziZR%XTQYqNQlUJ$VJ?o~wM^eA$c+XVj&-<-|77|lMJaT|4D81#T}W!4rx#kJun}~@K$C>Gz6g$ zMq&f5E(ItKEg2}WZRZpw@iQlcEXVh#@DmCV@4$c6eC zhPAkY-!kKppd>nAI`-kI%=mD~idq<#I7KT82R6%bn!byCPSrilb(G+8_8MkB>8>B*cbi-U6!E2c% zQIH!AFdXY}75~XBO^QzRN66fMRHa$=HR5GAlwMBdVb< zmf$o#$*hcxf@qF$*or$cs~pmxB6?sxj^VA$>Szc;BaFlbT*qITHOWy1oiPgs@j_;8 zMC3qS48dw#!cUoXiBJOVFco|7SY~|~WJV42$1T0o!p;W@7;9P#L|j z5GU|nW>XC0Llca~Cft6w0W`x{Y{6}rT^6ZP0o^eV zNAX5xcU0s-LyW+BT*DukJ;_iSoiGyz@LXnZ1Y}1Y48|&4#1EN$2~iwvF$KHvNM?U% zWI}cH!&02VXPEMHkG* zA-t415(zm`4@0pAm+?#HXkrATJ*HtVp2!>vi!7*#0a%W6_$qTe9*Uq9CSnKf%bW;- z^r(W~ScH@KAagP%@}nunU^8yXoH9s-^5}-SID*$Qr=uV@8ellq;VS-t|mjK1k29vQ14`t4WLPk_WUo63Ce3H2k8wJrEq4DnM=_Sghm*N4Y-cKGMAI13_4>L4&sH(m59iJx)_4hxP+fFR}-NG+F>g8 z;IYiLFvyG==#OPMi!USGwz;tGDt+)09x=z!_iho>@k!yzkbVIWrEJif`?i;tpcjY-&v z2Qv3VA_J4&b@WlL*L;Iv9*qxQHJzPZOdz+F}ZJ(8z@9=!c~^gU>S0Q8OyoyXjKOBy zlKE(m3gyuab8!T(Wj;kgZZyDftix6OC-XTeN}(fWU_YM8dYHxj9;=*6C)VyF%5h1L^fJjWI;^~z;c|!SJ~+C zPz0?o5j${SHbw}fM-}wOBAmnr*_bhrA5Ad^n{i7vmO(0%M>ovH5xka-9R<130K>5k zSMi^0oTMm)j+lY{cqSV+JhGuS24N*G;Ja+R1Sp0!n2cR`C>uW%GNKy#VhK*;lWc<6 zD2V15hpo6Ho6sQ*DxwGG;~3t`CW?k2G{Q)1z;*nUO`IHM&>6FE5HDntL_`kM#SpB< zCH$04ng}J(4pXrQk7bjEL1xrIe=NgUe34Ba7lqLh6R;ikWK#r?4wcah3vmMPWmCpL zJ~Y8-Y{E_1R6gb21 zID^l!+2f!PT3|f3;jU~BkF=+k6cqf}PI`X11Mqwjv;Gb-+6ex=>n2ketDVsYI za-trFVht|imu#NI2u6EM!(Kd*4GN1asEGksj&t}bn>QYcpcN)!2ky(}3xV{gg5Fq! zllUN;KPK{{DaK$kZpjufNQLs~hPgO`*RlnpAU7IdIM(4R{*x_~6s6D+Gq4}eWDAE! zHq^!-ti%O;mo1V2#n1+mu?r7ni-tl*R6}1Z!D)PwEfyOE(H!Hj6?bHdJETEH^uT-^ z!&})B(GY}27>NzIj=!?O$x#NKF$)LrLbhZ?PuWt5Py+2R6?^blwsaU| zMh*1GGMvR1*)nla7%edY+i_2}Yyjy{8NIL&C-7djTnywx6O6_t+>|XZq(nJ%#T*>Q zE7=N>kqh-P3~O-(zhx^XK}mGLbnL@Z*-GJ%6}2!BD{vm)WGlxXyp(Ma2{};@L$L;z@k_R0Vg#c-reQCh z$TkX#EU1YASdMe}D%&_7il7xHVh8TaHVJ|BsDj>Dgp>Fn+cYNfqbbHf>PAgsg%e3xyN0L9P-ld%gA zWm|_rMpQ#zEWv4fl5G$c!53k7YQEFS4EEqA*%w0=DCx zY?lDip)z`5Ax_}EY}Xjbhb9<}O}HuBO-PAy=!!Wwj90SVBO@2;V;I)r3VzG>NP?2+ zfa%zWr?Nf6AuDQOAXeZ!zRC8AkD_ReN!W=8vb{qh1FE7A7UL8?%Jzwc0%(S@*n-=# zeJxU>0=i=!j^d4Mzo^KAh8ThMxQ0Kn{ga_II$rf8h##^86QVfU zVhVQSk?f$*$b{=2K%sDz$afa7>4J2X1-qA^BcBW~cI z?64Fli!PXrLwG4WJQ8xE9)@BKF5{Q%h{Om+drZS#Jdqt47Fkde1F#(D@KttHJQP7I zOvDb{mmM7f=}`r}u?Q#eL3T_`2#;*2jX_w63-~TOF#(F94JKn39?DJ%g^Z|%zF30O_#`_yHVUFS z#$hY&$WC!cgNo>Z`8bBRvQwiW2#qij8*m+eWv3-a8Fa=h9K;LR=@F3wbuk31aS1F!DHE(VUQU$&>zci7GGp%#YJJX!~|@|J=xg-q(f!&!a|(Dd)YZLkPl5T z8k=xacCL^T<&W3dIdWtUi_Mg?@oJRHRv*`-mD2MsX- z>v0W#WS1pFX>`I&9Kdtg;2A^eD z#X%voz<6xKUD?$hX;BG1u>i;MPIgUnCI(zbwkOmde1M_hVZ)NvH zLl7EaBsSnW{>tu6jxy+sSvZInvil+;2kK%7R^t+W%I;5u5@?61*n`Ki2f`pTYM?)s z;Viz$9*m2^Xo(5fj(f6)0!W9-=!Jzif%me9V;~=zU^F)2rtA?RCCZ^I=HM`1$sUc2 zT&Ry>Sc@z8Eqg2pN}>a%V;`Q%9uJ4CsD*)8f%Et#dm=uHqBSOACmzV242cY=iauD3 zQ}`%*Di#W$8OCA@Zp)swNR0~Uj(IqWH?n7Yw9hSKPSnK*#wvgaZo zJL+ICR^cLk$evG#;%JL0*o{ZB7eXTws-qv4;tW2^UW|i6Xo2zAhP$$tJkp{PdSU^N z#6$ccIwiZ!^5U$WN{BN**54SVrK_Ig-k zK}`(6a-73g*&Fdt1g$U;J8)n2W(cH574*g;oWuv&TQQLzO)&Q2~Oja?8De7h~^lFt+*rm$RQ0Xq6g;V7~aZ0j)ovK!bohub^Mimk{o5w8MAN@ zFJzxaL=M!&5Uj=}{FHr`2qn-CQ?Un+WuJ#ZX4F7`EW=rRk$n*th0zidupRegUj~p4 zmC*|eaRTpUU&TN^G{I|@t@LTq65|l&-OvgSv zm3$AuP4<0!6h&)H!cIJp{SXovP!)Z!7^m=2_G2s*Kr@WR7TlKoWRV&b z&>iz|6mMidM@1eq#0ad%HT;qNk_@HM2{Um3&t<Y5XI3JQ?MJ4 zWWR?-CR9g1EX5gomi-Y2h0p@yu?=@+e|n@vCG^As9LGD^U(t~ljWG%vaRdKkf2TlM zbir&K!b{ozA|WU0VJOz%GJeVaNsM5$$29E46WPCEkp(p|0LyU>UuFNrLlLyXMC`zQ zIfX!aR6%bn!byCPGcl1LO)&NopVu;8x1fV>u?qS z$@!!xg^rkk{dgu92#;*2jX_w63-~SZ`8bBRa$%w&2#qij8*m+e<-#UM8Fa=h9K;K`a1oIMbuk31aS1== z!Y4urw8K>F!DG1yVUQU$&>zci7GLBd#zkSY!~|@|J-J8$q(f!&!a|(Dd%4IlkPl5T z8k=xaECW)a9J*o-4&#+v)X2z%`WS|_xPsqu(UPDfI$%2X;i+8oaL9^U7>E@(k8g4@ z;-e^9V-j}afn3ax$bhQogT*+7k8-hMp#Yj;EVkgbTx^TfsDSR6hog8S7bhz6pdm(J zJ+9%8T-;{kZE0@S4Eh?cW7T`GE$t8}Cyl9M3*oYhWCzm7z%AyNq;}BlTC5?ofsE46g zgUk3Omn<=Y(H_&V7fu?qS$)!z-Qs{^o*pFv&>B1u$YGV*q z;sUOKwS*MYFxrkxonA00_`vrd+=B;dl+O!4fMw{ zoW&Qp9C1+?EinPxaZfI10O?Q}y|54`@Ln!g4CF%-jK(J16h|;pq8z$n4i4j$T%O3t zh58tVwYY-cazRN@5*;ud`|wmQZ#ZN{Eeym8oX0o0eDP5ftuYBZ@jxzrNMt}&^uc1B z!biCRu}}cbFcw>ITdtr*YE(dX%)?Qhb0fwJg=**Juka-||6C+cA+*5ERJ$(2rwV6?|H?8OtgGGUPg zH8B9oaSmVQ%Em(xw8BK}zM?z5+CHs$3%WK#TabHEx8H?sZbu>Fc(Mg zTCQRg;U8iLRWBe4P3@mH>9a+E=5%)&vu zkgF9DIZzivuo{=}Q?7O*lt4R7#U4DCs}ly9Q3L(43}^90u5MfuMoUbPEG$b;hD*9kCPT`|mvsfsAW*CbtxGmS*A~hfRnQxYa1tNny2V6(G{qQf#x1$-2B}aU-7ptN@LH}%6y!z&497ZL#eZ@=lcE$l zVg~l(nOv{%$cEY&gq65}?{d8ppcvXwa(!c?Aev(ww&IRl zKZi7^h#r`aV|XjqKN^D22qUop*YQ_wKys8pXUxJuypS6h5jjv7L$Df`@KbJ3B9uTo zOvN5NmKz)fnNb7%u?%PNMQ%u36h=!-z;@h|8yY}5R7Ni>#0k8Y8x{lk&;+Bg2{*;l z3n@_!T`>oT@k(w)WaL7948vMn!Ed>dNl+3UFdh5wRBlu_WJN6u#0s3pH@VUAQ53B) z2|MvXZcIpIKvnd?Vw}Q9xv{ZO0L?HKTX0)$oJDF>KzGc;QM{2G9~F7f5F@Z2*YHPf zLNb&_C(OhFJeQjo0ohRpgRu%1@k4G>LKH_^Ou=qElA9bFnNS`5uoP$TS#C-k6haG( z$2Q!Ro9dAkmCzFla2)UCrbS0yG{z`w#0~tDo1Ow?(FL<{2ruPkL_$u~!%(cjW&Dzx znHa%nk7?M8CvvmGA`5C_0G8t%zRJyxhazZ&iP(Yra&tl;J*uEL7U3j5$jyz3{Ah|X z*o<3p^9)jR}bi@qo$1}Nw;gJosF$gPh0pH~o zB|tH>!DQ^hL%GGFkP+3;7fWy&pX8RrMnN>kIBdloxup(iP!T;aAII=kZdo)0p%F%6 z1Fqw*-16iogU*E@(k8g6D;-e^9V-j}af!yYh$bhQogT*+7k8)dL zp#Yj;EVkgb+*XU!sDSR6hog8Sw=F92pdm(JJ+9%8-1cNBjZT<}19&dCBLcFc4hCZt zF5-vW&V(qAwwQw5cqF$gG%}$&`e7-~;IrKBI4Fb`7>{kZE4RlZEh?cW7T`GE$?c7f zyl9M3*oYhWC$}#J%AyNq;}BlT?T>_(H_&V7f<94hD8?C!~iVE zIee8n6c0tv3KOvd_vH?UKzdX`Z!E$|e2_a56Zz2;W3U;wu?qS$sJFMQs{^o*pFv&C&D8eYGV*q;sUzu69@2I?tTPhM;#2tDqO@5xd#bR9BnZLyYWcwVQ6GR zb@anhoWWNzIj=yqWlcNkeV-^nLh1|D@$bq^Tg4MW$pK{+5p#<7tD)!*9 z+>bEGj2h^VWjKp3azEpuFj`^)w&R}MuK?1aGJ0VlPT;-V?-DY&-a{t01D{5gNR^U9o$tymJqBSOACmzU~ zkjQ|l=!3;Lg^%(!77Cyl#$pR@%R7tIsDSR6hog8S@1r6Q8e#<2;~M_R2a=&QI$rf8h#&GH6QVfUVhVQSk$kAo$b{D_k zsDz$afa7>4A2vGjqA^BcBW~cIe7F=Si!PXrLwG44J`!@G9)@BKF5{Pcgv1C&drZS# zJduwW7Fkde1F#(D@KruiJQP7IOvDb{mya9*=}`r}u?Q#eK|V@MZ`8bBR@^Paf2#qij8*m+e<>Mts z8Fa=h9K;Lx_z{rF!DIP^VUQU$&>zci7GLBO#YJJX!~|@| zJ^91|q(f!&!a|(Dd-)_WkPl5T8k=xalv&W3dId z<v0W#`I&9Kdt=^bwF9bubvKa1lS`GbBWD zw8a$c#v}QRp^*vI(GN>;2A}0K#X%voz<6xKUHQx&X;BG1u>i;MPCiR?;Jtjw7|4ev z7>!N1DcUTM66Mepb8r~1%Z5W%)WSflz~R9d$4mt8fuN2jKW6Tz(4s~DNq(&FdK*PQoeR1xM_9k?%FKLpaF3VLG^PU3@n zgP6#VrWk|GxFz4vAQj4^8|LB&UduO%g4}3;;aG>O_)orZQj|hR%)ov;lW!6p*-#sU zuo4&WUA}1o6hj+K#x6XRZx#v}Q4M{u1gG&yzIkjEM01S8R@{+q;gALu(F5~w3~%LI zMne!9VI(%-I{wPHN{%wN)Gisne zmf|@t@LRrf5|l&-OvgSvmG2S`Sy2lEu>$AuO}=Y<6h&)H!cIJp?-mjnP!)Z! z7^m=2zI!YbKr@WR7TlKaVUZdY&>iz|6mR5vMnxVp#0ad%HT;q9l?9Wev@@l1Yncw|Fu48lrWz<2pE2~Z4eFd4h>P=0JE zWJERe#S)yxC;4%)Q4q~B4qI_Ye!N2(R74NV$1%K>pAZc}XoQj2fa~}xKQTGVpfhIS zAYRB%iijMjiy>HzOZX{2IT1>r9j0Or9?MS&gUqOb{#b^y_#!_wE()V1CSW`6$xjO) z9V(+27UBfn%TJGid}xBv*o2#+y%H%=4qY(^hw(~&W@O|-eGJ1|T)}VoSxHb59WWjH z@Kk$2ZON+7x6=WaY7VFTTH=j zJd$4$8ktZX{jd~g@L7Io927zejK?wizo7{!y*f6VgQ!o9KOo0iH9O+g^AdK z`|@i;AU&#}Hx}U}KFF_&iTr4aG1!b-^6L#!p**@_E{@=}{DvsVjRqKwb-0TEP&=*T^8lU91#zsLj$2e@o z9rkRL^*WD92~|g`2&%W3-vJ!YjFj?zP#T>u69@2I{$vDXM;#2tDqO@5`BMo|9BnZLyYWc=bZBHkb@anhoWW=LGjUJ| zEifM2a994UM_N=uPb|Q3ypul{9eL3hqp%S-@K64H3Y0|`%*G+Sl)n%OIZ+Qou?Cm% zOa5YF1fxBsVK1J@UkZyXsEGksj&t}be>onCpcN)!2ky&X34!#eg5Fq!llUNiH74?- zDaK$kZpmLWNQLs~hPgO`*Yek+AU7IdIM(4R{*%9v6s6D+Gq4}eNzIj=%EvlcNkeV-^nLh5Unv$bq^Tg4MW$pYjhAp#<7tD)!*9{G%|)j2h^VWjKp3 z@{i-9Fj`^)w&R}slK|48GJ0VlPT;-#(-_EyCK!!PxGDckNQrXjia9upSMtvzBNysp z7}nwne#^f|f|BTf>DY&-@-M?7D{5gNR^U9o$-jz^qG*jt*og=7uR|gOs-h1T;}ky1 zzlntcXoj)ag4^)W;*I>fsK|qc7=iV;hClM}lc6*^VI~gXx%`I+$c{P~ zj8(XZAMzg)qBz=O3U=d>{HM^!gzD&rr8tAn@}J|N5L#e7w&AY)7mu{4gq~P{<9H|k zH9GR5F-Bn{Zs4E%w-hLgE|`r&cq#uq5^|y*hGGpaBTqZ{Vp2wuzo zje^{0fZ`-xV+kPz-G_8N2XM0UHV#Q4M{u z1gG&y0T&wu(H!Hj6?YWy4rx#kJun}~@K%99Gz6g$Mq&f5h?oQ=(E-!34^I_{ z6b@NY3j?tN=kZN}$njAWtuYBZ@j!tnA&~)9(Fcoh3Lh1S8Vd!`3}dkcw-tzHks1}y z9rJJ$Zxo0g6?xDQBd{LV@JE3d$xs@dFcSyxT!EMokR5d}7^`p*KNN_S5XI3JQ?MJ4 z6o?%fnNS`5uoP$TS%EllPzWtB9@}tNfw&%NQ3*Y<0LSr8fq2o87mYCr8*u~w6o{V! zWzhw*aR@IJNDv7*Q4d402AAjP{s@y?CNPqOiz;$givJWy zkrbuS5i_tK&lE@*9@$VEgRl}8@Lhpa2~Z4eFd4h>P=VB;kP+3;7fWy&pA<+F8wJrE z-#5@?61*n`IkWD0}KsDb`ihO_vhK<2n8jFy;y?YO5vmH^VBGJ0VlPT;))Sz{m{ znqV|G;ied;L`sxHSIog-yiy>0WaL7948vMn!EXg}Btc1Zz;x`xQw4H{LsrznK&-%d zd{ZD-d=y1%Ou|k)P#||mWI$E)!D5`kM+Nf4LIE_xSZu*<1%fP6qXN2P9**LT0(qk% z4;o?w*5exfD3C81N~05I;sBm2kUs*lqYegR6)xh30tFJHIND+gcH@x(1w$hfs-qv4 z;tW13P$&)xp#{cc8}2Gl*dr|}p(hsLINm8xBs%heudaSHF{R!fG`@JBGVeX$6~@JeotgeV3-j712p z;+Nc-nNSTqFc)EXDz{cV6ht$Oz&c#OSGl#*q7u4b20{@jw@ystK_d*tN<<(^Zrv0p ziw>BKowy^no`W2yi~d-GaJ-RQKQT(64aQ>&uH%o~23b%8y)Yk#@my|04-`fVjKT(7 z!gsli(!&RVn1%g#B)4&Fyap*{v;8BXGzT;HT9 zg?5;TZMZ3ZN)0d6LLV%|QM{Df+!IC73S+PdSMXDAi;Sp>?wEsvcp|rDToiyWhGQ+x z2H8*>eX$6~@Jep4geV3-j712p;+NdsnNSTqFc)EXDz{HO z6ht$Oz&c#OSGj%Dq7u4b20{@jw_i-;K_d*tN<<(^ZvPZ0iw>BKowy@+fP);Ui~d-G zaJ-Q_FfmG?4aQ>&uH%o~L0M1(y)Yk#@my|@2MVJFMqvXk;k(?y>EVMw%))*=k~<_e z@}UWaVl~dVhVQQuH327 zkQ4PV084QKZ{eX$6~@JjCdgeV3-j712p;+Nb7nNSTqFc)EXDtBQ#6ht$Oz&c#OSGkMQq7u4b z20{@jcX3SQK_d*tN<<(^?vfNJiw>BKowy@+se>G-i~d-GaJ-SbEHO%;4aQ>&uH%o~ z8II&<6`~6ffnj_e4>&!WeAA75tRD zAtS1yJLcdZp2*!87X{#p;aH3F_#$^x8dO98reiN2$PI~s+-Qg(tiWk}kh?iK%Ah?a zVFzx@-C~g)bz>ttK4I0Q3+iz1EGkNdpsucpb-XRB_a?d zH#`N(q5~#lC+^5S;UEX-Z!0R2I}gFU-ebJePag1BKB7 zqp$&&@Lg_1diWp^v#=kJA8lt(8_#cte_d(K5J)W<+9!%4i8 zdp;>jp&ce-8*Yk!p9C+|LLV%|QM{CU(Gx|{3S+PdSMXErrHrVG?wEsvcp~?5ToiyW zhGQ+xxi>STI(lLr4&j;HTk%l{%`p<|aS`9--cEfw>67 zQ@JnWp&*)J1lHjKzRG=-7M0KiGZ2bMxvyg)4;o=GRw4pXa^IvtS#-c;?8F_pZyn@7 zUG&EigyW6ecZpE~Z7?2Na2jOTJcc%U#^U=%js628lgN)I0dVixw} zk=&25kq=EU6svI-pTfM~?32t)N$s!!m++k|J$w*|S=f(9l8udgXo8_wjkEY9IjK+{ zoiG)p z;H{J-2}+_ZCSWUW;IEW4D{7)Q7T^e8NXZhQ2wGw^HsUgVNXavx3c6u74&bqrA`bGS zDTZMU&f&9^GBqloGp1n=?n|kn!y64S2+MH_@1@kqP#XRS#&+D2(imhzZS=(=9K$Oq zZ9)`-AI2gCSMlq#*NUJ2x=EKwN>>d%Fc)EXDy5Hyf@p>jScePvDrHEEO6Y|3GBV|d95@>_**n;c$BW2Bk8t8@j zIE?3#mj?=?1x8^5F5$bBEj@e?h*{W=M^g6K$cH8viq$xaPg0IlD34Bt)EzaYMlrIe`A^_8|7Z0TTF_0S#5rh>ujSo_R$RL61rdpLJ=udjfp&Hguz&e2t-NM zQlKn4U@~^%j#S-24%9_|EI~NlNHr3p1lnLcw!E3n+UNhSH#M%4{gG;BK@If6d>qDe zsg?%{qXkA`1L(Hacd2%I_#hCoupf`4IR;BDUeCM73(*g<9x?g*b|rQbSJ^MJtTKCS1W!sZmB$MR&}>K|GNf z$3+47VmQ{~JibUx(x4&&FdcjGKx!HTxzP|oSb@{{Ao(Uo8MMbF?7(fQnMHQgK|d_U zalAekdt>8&-84@mH7}0V7>CWchTl?)%&3l@n1@4nCbf)@LTHYWSdWYNCbde3%IJ!j z*oTLbUo7MW@9j4Pt8fM%rPe7?4jnNCyKq-(6Ad|04+F3iC-7Ekn*=4%789@)H}IF~ zXI9iiZ!EwOypa48pa@!GG&bTgen{;zpbENSHV)vi)FBS?qbY`A4bI`S)G;+Gpfjdn z5AI8yqQe^vFbKg@YLoTFA{@gjsY^l>gCE8s z1XuA(>Y54Fz-x4!i!eNu0^^||nqdUi;R3!&-O{2Gx?l!E5h-<#i9Be8!B~k1L`gkT zpe#CIGIrvQ)YCx@)J1y%M7Y+F(4k;5zH#!iFeY#q$q`U z9$wtT{clVgIFW1{Zc0?FL0+hZK3IsOcqs*Wq9|Hn3^w5keoBKgqAI#$4i4gpG$bwx zz!$@@7U%Ir8kzX;fxZM^DVdAv}{t$44PF$4IQlMSPRSq(fzN#Z2tOLuqU*J2ltV{M!7kjD#z#X=)WZNQZJpyh+y3ACoAD>e-bxdapd{L20=D7?{z?6r2D>&=RAu5ts2pnv?-m&<(S30FR}~agZNPF$`;P4xgndsZjx)F%5fg zUz!>n-e`b9SdLS8FHK8^((p$xw&RvG-5?ukqc0ZW7+y&;5~3LVFcu-WieJ*qOsIw) zn2Rtxm1f05K{Ue%tiuI-m1d_!C3L|Igd$R!6BBvR2!pW_5r~rJra)PAz+~*io$SF= z+1LN=W}YL>%YnM+k0l7l8)<%Glt3Gd#}-`2A8A1r)IcxH$6-8|7J8sCT3{45;1a$| zi_*gfftZE;cqA>3jeKZ=p;(Qx_#`b!h4Scxso0Hs(oz?>P#*)a3@7nUT9y>0&<+!^ z4L2n!)p9S?LLV%|QM{B^c%mp;VGK6m3Vuo}GomWGV-61DiL@#%3cwe`u@>j?MOvK( z6%l~x*oz0!ni$B9h6ut6oW=)fZSs%-7efAZv$hObdrZO(+?Li^WJewJ!(trAYiWHV z6h~`}!)9E=ZDqdIzG9uDD|v@t#kp*cokJuc##v?(1bqbp`&A0A2}v5*&yF$Ak{ z1|Ox(DNzm`F$KGDSK1N{IZ+P-uoNfoR@#~bCD9fWuoXA(SK5{pHPIUja0D-;?Fmo> zEioD!aT!0P9T`vs-7p&m@L1Xz2l>$y!>|VD@LAfG8WqqP)368krQOltjRy0pwCMD& zo85!Rmg5xOOM8-`H2e{a?YJfFHOPkA=!-=-hF4N(LKK4^#)9rcui}@qFB7Vv2j(IS zPo@3wP!P>90_$)AU!?4KhlvbsDWOXkHdH_9rZwAw7@8Az$JW_j-`hW0x=8w@klxz z8~M-#L$MlX@kt6#h4P@2@Tox=1L*$0V_NubvU}1A7r9U$1F;Mz@lHCK6s6D(6R{09 zCF;~EFVsRGEW}Z~lumo1C|Y3*HsK0>N)Z`R72Poh2k}HY6Bh;Gi{V&{^Y|j2O@oRE zz;x`z1L<50(ZhZ!EwOypV1sKoPXW zXl%q~{E%*EKoxYuY#hL2=}sKvM^g;L8l1yt>27LNKxa(D9^9AiMTa*UU=Wt$6y8hs zlc6;H5sdA)B|R|6hT75_eS{QimUpG(kkTt?!tV9H&q^BuR79B7dJ8?&P<{$^^qCb`( z9B-uOiBSS=FdkcQ9e<=3Sx^JLFdv8UTzct&!f1g}*nms;F1<<*9|U3+_T!QCIyUm5 z35H@d&f=5wCKbw~6Q*J}_`Gjjt>Oj*(c8i})t}Ooz(oikaAlhtjWD$cx4pf>k(!kJ9gyD2I-if?c>P z{fUO0sD}YqiW7J%{Y`?BXp0HhiW~SVqpYZj-dKPmcp+mFpa@!GG&bTge#qDisDf^o zjRSZr)mOM$ZJfXUd2J2LSd#fB5t8o^eWD=)Bd33^5?8ZHrBrbBHJ_ceLPU4+R(xfPbc9@85xG9s2w@v1STIhp? zIEt4t$vsgNtuO|ga0NeQQe;F`bjKVV#1om6aZv!i7>>0#k1sN*(x4&&FdcjGKqhqz zYyJM<2YW+q)UY2XpM2$jBEHUlRh)5qbKI! z5T40oh>t>Oj*(c8i}=>nCl=%Cf4j+;P9|ezbj3{U!$X-&v5*&yF$Ak{1|MZIr$jk) z#1!npU70M=kQ4PV084QKZ)LJ3K}od51Z>3({FU*_ikj$+1vr8iGT9QK2wGw^HsUgV z$YjreD(HsUIDp48IpQEcnqnB%;2b{7d3hV(`OQgy1TE$>hz1YUqKv2*XpEeDP4Q(z3<3{&kbD z8QBP|!v%bm$)6UL&;>IPib$CPF_8z2Fc>Qlfhd`RDNq(2Fc~{>N2ZX29H@)_Sb}i8 zktv)QCC~=ru?5%hN2W*?)IcxH$6-8|De8g3Xn|4KfJ^u;Q!G7v5QtgWk4G}aV>RR#cp_6SE(*XG!?70U@kOS58dO98 zreiN2$W(}d+-Qg(tiWk}kg1p)WzZgzumiVcDp_Pl9rVLu9LH;!%85`MtuYRpaSgv^ zd@`dtdSV_9;h9X8_$Y+t7>V_`h;K4g)1flDVkY+Cp-i<{$cx4pf>k(!k22L$q8vJ6 z3U=YHOpR#BiFz1-r8t4NGBuN+B-&yEw&Di<%GAn=n&^!MID!{4wG*HST4FRd_V?Rd z;9objFO&U{sgnU!&<(S30FPzr#zB5G#W1YFIeeC>ml_q&8Pl)__hsrwhc_Bv5SHT< z-pe#dhSKmyFt+2COhbcgsExi@gkyLm(N2Vp$Fz73{Pd6#6v+e z!w9Uy1$>ohniiGN1v3zeNEzRl$b&{0jFpH$luWY}D2oo5jGee6)7(K0)J1# zACF{O$3{Lh!BDKmS$vXdlM3b02~)8f_hj0-$c6eCh-Em5cQWmgq7>R;BDUeCj6a*{ z?}b|EgM~PXmon`=Q53B(2AgmNKV>>(L{)Uh92~?GnT~N$0KOQGwK$J2GM&<(A_6cS zd+|V~a}4B0Lj++3PUC}2Kys8pdrZO(+?MHLksWo=4~uaeuVuO>LUFXlIBdo>{FVvK z{Od>Lk$>F;RwwI;c{qe;GTq{%5Sn8o*5e|+$#hSL%IJ!j*oTKQJz^m*8e<4n;S4^? z^h}9z=!hxUg}XAnq9G^hVE~rm1m4Q@PJ)tXiwW3@8~7{JCo5{AHx}RsUdZ%KfFfv# z(b$N~_#x9T1FE1KX5#=J%k+gb7iID}_1v*V)>nqwr^<08Jv%t?pJ=!%)xhletAV<9gZV+dB^3_i-t zONnymh$+~GyE5~mAt&l#0G8qe-pVXUf|6*93D}AoXA&3wKXZWve`OYCMNRa^0vy2$ znMDau1T8Td8*v#wWEN*Y6?DUF9Kd6lC2^1+O)(5>a1NhkmZnApbjCF7!F`!!(cz5- z7=-0Gh4(Vclc6;H5sdA)C9}dH8)~C37U39P$*fF>V(`OQgy1TE$*jtRYUqKv2*XpE z)$vde%`gJ%Z~3mZBOjVzC|2VvKFMrJh4Scxso0HsGFx5bLVXOxGMvOanQci?3T*$j ziP(mlGTZrB+r3Z=eXtNm@ls}oCyJsK#$Xe!;HS*ajHrt4n1h3OBC{(l3cwe`u@>j? zMP_#zR73!#V=o@a?1_QgXow)Jz-fGt*_#|?&>oYp1Gi;DEwZD|z9Oyi{OcyPAK79Y z$7`8=iBKG^F%Fw?4ZmgfXGV4O#5^3rGnoVNQ3%a366GKW&496DkOcHyqf;b_Q-dKiGEIDxk^N0Oi<+F}B>;s*Z89Lq@kRMGk3~O)>pJh&@Mg?@nH0;5BnUm4s zjRqKm67Q<<~zP!P>90_$)AUuDjvMJ05>41^+5=6p=#K_d*tN<<(^=0XaTMF&jA zPTY~X=pYB`qCb`(9B*VUB}NIf!FX)Jb^MXJoCP(|3-fUp&ti#ygpt zNl^;zFcI5uQ|A9K?cDN0E%d=c9K}nS+ny+jRv3d#xPqTDcQT?Xx?>Iw;)%@NxF`T$ z498lW#}}DCK{w3C0X&v@69@Ux6vMCv=kQtPZE93NXH3H$ z+?RP59o}ewL0FDccrWul8A`(+!Pt&lG9L`Gp*H$r5su-NOjJS?gCE8s1XuA(=3^#Q zLl4YF7@o>}iid(|h7nkY3-~JYIV~!o3uYh`kuqOmA`cp2FjgW0Q8Hgsplr!JoBsbr z^FMQeuN}xHV<+y&d~=WkbjOQ{xJx~}e zFbW%R3EyRYrH2m!F$??gNalBJhIhLdK|GO-78eEJi{V&{ z^Y|j`ra?snU^@2Vfo${`$c=^wY8r8S&i}fx(N~b2#s}FL$x#OFF$p_xTQ;UecGN*X zEXHxXmW`DN#nBq$uo>6zTQ+uPR7X$D!y!DAjT0Y*&>SPN9vAUVHf}mpMpw+lK0K6- z7YljO7(=iMXYf%reoB->M@+#k+?Dl+hMcH}0a%I?cq^MA2}+_ZCSWUW;IFJ_R@6jq zEWi=GkWH8XMbHwXu@RT?LpD(cR6#e)#sNH*O&kaL(Gt#OK$L9S6ex=h zn2epcBb&}a4%9_|EI~Nl$fi$>5@>_**n;c$Bby-$YM>Y9<1n7fX7oT|w7@8Az$JW_ z&6FNK2*fPx$0OOyv5^l=FchnC7M~u+i{yFvf5tSMC6#QJ^5}%A*o}L#SzY8peGJ4h zoWwg>ucRo2c9@85xG9@W@Io#0!9pCxOWEw6D2i4XgH5=CpRzeJqAI#$4i4gpY|gkS z0ACEpTAarh*<5K*5doNvy?7w&9Rs=15J6ah)A%5pJ2}drJtkoXZp-Ge$c{Sbhs8LK z*RpvNp*UJ&95&+`e#_>|jOyr#c{qe;viakq5Sn8o*5e|+$reb5%IJ!j*oTL*1!E!a zb>Bf1{&iEZG1(BT!Wn#&EtC@F&=FIx3wLD;M?+53!vHMB3A~jpk_08u789@)H}F@s zXjarjZ!EwOypS!H07cLeqp=Z}@k6$F22?>e%*FvcmMswn`Oy@^umm%`eG4|;gxLJgeV3-j712p;+Jf> zOsIw)n2Rtxl`S6+1-ZyEB@1ex7v|$Ip37GCKw-4N zC~Uwbe3z}39zF=fEbPZ4+3K;84^1!>t8o^eWNV~Cd33^5?8ZIWnl5soJ_ceLPU4+x zt)wW0c9@85xG7s(@Io#0!9pCxOW8V}D2i4XgH5=CpR#o`qAI#$4i4gpY`wTB0ACEp z+5oS)MgMhE?>yNT+4^Zv5doNvy?7woAO>=yA%d_1r}06yVRDo~drZO(+?H)*ksWo= z4~uaeuVou2LUFXlIBdo>{FZH!8P(Af^Kb~yWShoEAvDKGtj9%sll4u9%IJ!j*oTL* z&0--h8e<4n;S4^?HcyFi=!hxUg}bsXq9G^hVE~rm1m4QFOoEbViwW3@8~7{RDl2ND zHx}RsUdZ|-KoPXWXl%q~{E%&(0aefqvvB~AOZnOr|GH@tN48CVG{rEi!8v@EZJQbu z&>7RP2lr*$MTa*UU=Wt$6yD4FCqrrYBN*FpOSZj1Hq=I6EW$ColI@TX#o&jr2*FkS zlI@rY)zAZT5r(I-o#LS&nqdUi;R3$Oc20{*=zACF{v#zsCg!BDKmS$vZ1l?vt22~)8f_hfs!$c6eCh-Em5 zcd~twq7>R;BDUeCY+u0(wa^C(aTG6Q`+1@$T44+};R=4r_Rol_=#Du!h$pfG;-Ub2 zF&t}g9$#b!ra?snU^@2Vf$X3d$c=^w!U~+m2ic(HD1-KxgdMmoJJ=#S>YyJM<2YW+ z4oQUKXpM2$jBEHUJ2W$@qbKI!5T3~ni;qHR?)Q95zJKS3({FNP( z6*bWt3vdK4WXC2z5wyf;Y{X^!kR6u+RnQHyaR85H$Hzf_G{rEi!8v@Eosb$8&>7RP z2lr(sMu#^VU=Wt$6yD1QCqrrYBN*FpOLme$Hq=I6EW$ColAW9o#o&jr2*FkSlAV$X z)zAZTJ?_LQ|F4@VVPsEbr^Z7;G{Xq2!v%bmot74r&;>IPib&b%F_8z2Fc>QlfhgG- zDNq(2Fc~{>M|P%z9H@)_Sb}i8k)4$oCC~=ru?5%hM|O4=)IcxH$6-8|o#TPRXn|4K zfJ^u;J2yRi5QtgWk4LieVj~}#U?^7OEI!H3PlfX6gsIq#d$J2$-Jds@* z7X{#p;aH3F_#(S34Jsl4)3Fy1WS7T4ZZt#?R^T)~$gW6^GH8!U*n!)!D=o644*Fp+ zj^nlLszfM`))V_`h;OoM)1flDVkY+Cq3pU? z$cx4pf>k(!kFx7iq8vJ63U=YH?1pH_iFz1-r8t4NvKy12B-&yEw&Di<%5KVvn&^!M zID!|lAqm1t@Lk^j&7nhzkhR2UY{X^!klmaCRnQHyaR85Hx5PnyG{rEi!8v@E-I^K| z&>7RP2lr*SMTa*UU=Wt$6yD2jPlnR)M=-YImh29LY^aUCScGGECA%{riop+K5rV7u zCA%vVs-XwwA`DMucgI6PG{Xq2!v%bm-IErT&;>IPib&bLF_8z2Fc>QlfhgI~6ex=h zn2epcBfHN*4%9_|EI~Nl$nH;!5@>_**n;c$BYPkVY7`D=Sm|Fk2YQjs$6-8|J?MeL zXn|4KfJ^u;81#ClxBH`(*)P#Ik@ z6Z`N`_ChS=MPm%XDxASb*^4Pr4jnNCyKqpJi{PMg?@n zH0;5B*_+YfjRqKmjOVgXJWv=d zFbW%R3EyR(riTv#F$??gNcLH5hIhLdQX4&0WFvdE4) z=!eBPj@Pmu6QMX-V;nZ)8h*=u%8cshiFr7LXR@E;qY#>7B-Z01zR7+`hsx-Rnb?Pi zvR`8%FB)SAR^bdj%6?0Ua_ERD*oC{Y-=iTX>R|wu;soBx{z!t7-t#8@`<{*c(Uxoi zw&Di<%KprXn&^!MID!|lzY?GbT4FRd;xc~7{?33Z=!V%ifXA|b;vheoVi?xo96rnb zO^piZjA__|`*IW=-e`b9SdLS8FUKT9Y4{@;+i^<{56T=C!W|a99ByhICa?|{D2Id5 zaT1~!{4f?FxQbtLqGdug^uSz%;i()q9txrvMqnK-;H#YIX;BGXFax28loKN+@}Ln0 zVX`w z1vStM^Klr@<;3+sVYI*~Y``UamlH2Nd=Q9P*pEkY;>SilG{I1;##wxlujSq5?Cr25y$0Y2C zDzNbWziyJ>mXpFFJL;ex7UMWx%So9C#nBq$uo>6zTTZIXsE(ePheLQKCv|)jLUWA7 zdR)XeIcd_NGP+_W_TizNw6TyEjWGnPa0Vacq)Ul%=!hxUg}ZXnM?+53!vHMB3A~k) zAqh&NEhb$j%j306`XFwHn!)zSDV>wykAU~R7 z7}nq%KFi6P8WqqP)368k<#ZcuJZOZ$ScwQk$;p!fWzhkXu@iUXIw;)$G+aZv!i7>>0#k1ujcr9njmU^@2Vft=DYkQ)sVgcUfA4|2*R zM;WxoB<#R#Ib|)fqYnCEF^=Q4oN|d!9IY`9n{f@l<&@8i>gb7iID}_%D#S-2G{;D+ z$3=XTQ!yPXqbp`&A0Eo76bpIL7(=iMXYf%@<&-Fgjp;H{i0Nl+4PF#%g~1ApaI&5D}njRiP@7jmj4KoPXWXl%q~{E$;U1FE1KX5#=J z%c&6u`Oy@^umSO zqc0ZW7+%S#n-Im|hp`C3Rs51uFB7Vv2j(ISPvz8)hk|H^5m<){_$sGCT2w+8%s?n2 zHq1bVFX!}oJJ{779B7dJ8?%&V+T1<7yYpW;dmpbNn(^h8;r*m zT*n_dO|zf|dSN~eVGK6m z3VzCIlMz+X9dmFHPvo?XivsY)aID37e38>G4Jsl4)3Fy1KHSRYBmL*z)A5fX$3Hh3 zA_yyR8Xx4ePmVHZk4e~p+j2TsWJewJ!(trAYdIYgp*UJ&95&+`e#_~U8P(Af^Kb~y z$y!>|VD@L5jp)ZPC`krgl)Gk7a&YyuQR8%)9u+!jCc zkP+3;2a9kVFJz66je=;7vDl12{FF5z4Jx8L=Hf7dWlfBRJZOXwScl8_B5P7|ltpLE z#C|-IH8~=3pe_buCC=f!tSO050_`vbyKq<5R4-&k4fMkjoWv_x)8e8qT4Fr5;wFB} znw}1o(G&9#fKXX8Vjv%yU=%jsD!$2@nG)sE6|)ogwC82;|Nf@UJiup9WX+0T3t3xZqad1NEH)z$KV@x8gNo>mxj2kqS=*zfo3N}{c$@8c_^c5| zU>z>wi>w{VQ5Kys6Z`Q<*3O8?fw~xsl{km@vUVjx3ADo$?804HySOIiktW?Yi~MKMo-K~077N$i-CM-f>GFjtN12se@c``SIou%Jdx!e897lO zL$Mkc@KM%*Bq)Urn1(&LFYBO ztRt~d0L?H4n{XXJWCf%~g{{6Hmxs3r=*DMra0t(29gT|IXo%rhi%a+{>sT_BK_|?> zK0K6lJOZ+#4hCTb&f=Y{6A4iqZ7~@;aYxok4>F-T`eHFo;H9inaZm^?Fb-RA1HWXQ zPK!$Dfq6KB5LsuUBQF|bB-Y~!zREhA0_D&Jv*3@%vd%?T9$Rq}zhym0hsx-Q`3U$nId+loHV;B&J&b{TXo6ALfUEc>>rqOSM_0_o z0X&iQI5KjgK89j7F5siACrMBW9WV`ha9`F_8(C2c1F#II@kZ9O_$Z3jn27DTg}<_b zGoUJZV=xvY>_D1c@dgH5=OAF@JIqXN2N4i4d&tgxuajfNPGwYY@OvYsbH8Fa!7 z?88G@FCrj2>R=F7;4I$BdYKT#(H4`j6L)02@*oqcqc0ZY1YXK|9S4Qb0^_g+H}GrK zhLd68ZQi7n^`;VfU>=SjMAqBr$cx4piS@XGud?2yKsj{5EcoNGtoM=Njd~b@RXC3i zvOXk6NwmjQ?8ZG=A1!1-P4vf7oWg5apW>kiT44gVAqanDeNK-m=!FG1iZEGUVj@49 zVl+158otZ=nhHMfg&z*$sjP2NkP8hk3~O)^pJaVciqhza>DY@0vVJ(ohT0g2}*EiQlrFbf}D;n2!L2%8n2N`OpNTumM-`O?JeTD37j~ zjRSZhJ5priM12g!YFxla*^!f=6gprU_TawkC^oX976xD$PUDU2sPR!0tuYbXaSMNC zN6Ub!=-psPck914cC>|jb_~yDM~{U9XofM^gzNYrJ4R|$KsU_6Av}{EGb(bUA%iqhza>DY@0 zvQs>C>Ph zx??U5BUpBZXvl*`7=d-Tj4!e?CP!Ix#!T$TBiWfEA_wYXFjnFm-pkIM2qn-CQ?Ltn zWoPk1X4F7GEWt^ zJ9m5(MQcpNcHF{W*?BUcDtcofj^VlNys=OK%`gU=a2-En=Sz(W=!Q8sglDqzM@4Ql z#Bi*|C481$AQ{S_6J}r^9?C8l0ohRpgRlZ;@lJN3geZ=-n2epcBfGE%nfBL>aVES? z;p%+W7mINMFJ%{rgF!*|)`Qo#ql@WVkom0dmxa-ji+VGSRnnj$x??U5BUpCTXvl*`7=d-Tj4!gQB}Z9w#!T$TBiYp>A_wYXFjnFm z-pj6$2qn-CQ?LtnW!LmVX4F7GEWt^nA}ebig$1!F|~cY-B|(48XFb zN1dDDZ5o{Bvp2FE#z#@K#zbt#E&P?;C&W3UO=@k4f#)Tn@N zn1e%jCc9}=*oix` zTY8WQ)zKG=aRM)8w~B*8Xn}Fqf*bfHyLDPrLJ!Qt5roKY6CHWc7$dPBSMXJK+Y~5= zE|>*>JeJ)q61-6lL$C_x@j-U`#3+gOn2O!Fmpsl1ukbb_VmkKXf$VM$vY|ExVmZ#>t?cdzPz-G_2|I9GY(YatR6`#u!g0Ki-7_`{ zqB+K5GXn8bcCR$3i0+t+!w8n$I~ww!5k_DgF5`>rKFLuQoiP*p@kn;xh{%Du7>t!T zhxfAkC5l-5&8F}+{YvmzJ50eY+?Czm3z<;^{jdZl@k;i9xG0R47>}*EiQlpZrbA`) z#C!xGRQ8}4$cH8vg$=lhZ?XrcM0s?@Y#hK7*+U{DC+cG;R^tLb${v~orO*M>um|^L z53`XKwJ-q7a2ju9508(cXpM>3j$8OEdqf6QMQ<#`F+7(&G8PJ;8OC4}uH%R7QK?Y@ z-7p7-@J#mTsK||m7>>2LgwL|aBtsc=!VK)gL)l{^AUo=m>+mIf_Jcik5TC8US-g`y zE+LAeEhb|p?#LeRK_*m3Uo6H6yp%m54ho?K#$gL?;Fs))X;BG1Fb_u%B70JFG4nmtuO)G5QIOnXQW3J^uhuhMVRcFF_9llF&Z0j4c}$YN(CSI!Vd@WRQBvB$b|+N zhBdf|PqO`zqBJ^UdZu`PpGyDtT)_6*%V!T{&vB3qwJ{LOaRzT?&rN`0XoE@Ef!kt> zDl(!P`d|@`C znb?mEMnUh0L+sDXZ1f|Gb9ds$o*MoWyxR@}sI z*~`HMk-auPilQ|pVmof(uk3XhP!+wg5XbOb_WD>T zfMytjO}LI9vNxng1$4t49Kti%8>1pO8e%xs;u1c~-jobw&yQ2gG{K7zF3SCcqw~Z927zejKdb(z%SX`)1nf3U>=SjMD~v8 z$cx4piS@XGud;WhKsj{5EcoNG>|K%G-}IRizHWAT^I1I%!77}`2idz5qa@m6Dt6^&B;peFibDNf_VmkKXf$T#LvY|ExVmZ#>t?a`IPz-G_2|I9G zY+FZ0R6`#u!g0Ki9S|D@(Hvv38G-mI`)C?eM0d=^VFb%Q77cmO2qUl#m+?jR@#HAG zsY<_{;cbq0=ChgDk4LgkL_`kM#bB(&IlPyBG7(Ck9j0Ix?#e#ph0LgdeprH&cqRLE zTogu2jK@~o#BbSW(xEbXVm<;8D*J2<qCC1{HV)v4?DLV46ZJ6^ zt8oDzWnV~wQs{tb*n|7BFWSh8S{Q(3IE^>5FU3buw8lhi$1VJoeK`ZFqBj=e7@o_% z5(@>;3}dhf*YQL4)zqkfZkU5ZcqaQ=ROCiO3}3pgO<;JNYis%J5^mt?4qY${{&+0=ZX|f49)@5Q&f|mZdx=pJ?J*U* zaZmPr3t3PT{jn6M@LKkRcqoEan1F2v!XMcW)1wM{VF8XJO!lLg$d9HNjg7d5@3J4K zf)9M*hl48~uC5i{=J8Y6Pof|f8ekaK;37WBewq}e(GkG9q%IE(T*I&f&f6SBX#p?Jxzqa98$gFJwjy^urRI#4Fiv z;-WBGVm!9uCVtC)n+}!H6Y~*(P}%QdARn4w6gJ>0zFmklyM1_@_bFw+FORO6jRSZh z`$J^pM12g!YFxla*&maj6gprU_TawkPd2im76xD$PUDU2&+$}B$74BOk>HJb7=l$ej}LOJ z#3+gOn2O!FC&#vs1vSwhOK}RXa-#B}V%136J0WJ7HX#B!X$TRG7Z zpcvX<5_aIWIO>LssD?gRgyVQ2Cq`@(M01SAW(4A=oS11)5#5_VDc&=@P0YD`b{N5O zVnst9G{Ok1!)1Jt6FWJ|qBCYIPSnRxti}a=l#?V0N}&U$VGr)hNopf2YGDAD;WXaJNfsYP(Hax69k=jTPVx+> zir!d=V|XqnMJyCRGmOEe@y|vz3~!U-I-mWJlQK0bpd04k5T40N6&1PB5W}$+m+)Cm z>SQQ`PMCpxcqk`L1Y}1Y48jVW#XC7^6QVfUVlsB(j+}HJWI}cH#bTVmOF8M|pb%PM z9Jb&Fe#yy@7M0Kg^Kb+qaxz9oUNpu?tj866m6It2%ApHp!5@$1WR3)H)WZ<0!g+j< zlO-`qqCKW!H}1*FY9R}1qCb}66kf~877sf>GFjtN13T zL`sxLSIou%Jdsl}GIF9mhGI1?;G>*UNl*$MFb#WfUruQoSy2lEuned1MoyXdD2mpY zi0!zAzjDfEKvnd{LL9?$Ipt!Z0GeS8HsLyc$SI#170?ZHa0t)j_(Vl+G{kVM#U*@} zQz043pc7_bA0Eo77y;Q)2ZOKzXYo!>rGzMswwR2axFe^s2boYEeX$rP@KR2dI4Fb` zdB%^865giDI6m8g8~7!sYFbo656r_6gvhBD9eL3hBe5P=@KsLr6ex!-m<4}4mQy1V zyipHBunOn#K~BxYD2eu%iru&;rPNl_XdF&%sHKu$vk z*-#q;u^eacR!*Y?D26tegdMmoj>jV-s-X`SIb$lN4sX-=IG??c(t!Thxc+?CPE3c z!xZeoT{*41kQp`54@+^kCqCSRVH7?+zoDNA)3LP*FdvIS)M;lpD3j?qWr}0Kkr}!v} z)|iOxxYg)Yyz${}I{lT?IRmPqHx}X;p3CVH3kA>&W3UO=@k376)Tn@Nn1e%jCdW4_ za-$)JV=XS>vz%_pPzIea1N-n$PWK4Njyf2G6*!A`a(X00akRx`?8F^8Jw3>T>gbEb zIDwaPdc{E@w7@uQ!43S9(>pCHp$F#S2twraiH^KzjFDK6EBGp>Zwiz{7tDe`9?R($ z3ErrOAy|d;_#mf$Vw6OCOvP^8lQY0V7Su$4EX66jmNPJ3mhpbK!`loj!e^~80oxFS zKXL}8M-}wK0vtt{oWU`XA5AeD8*vTaEi^B+(GbS4HpbF&hW)M9$R6$cg$Giq*J)k8-9ZK`C^=H0;5BIn!-qMJ)`# zGMvU6IWyv;C|YA8w&NE5%9)t~RnZ#@aSYGp%!-8qXofM^gzNYrXLf2-KsU_6Av}}g z7ZthD5W}$+m+)E6oMb42PMCpxcqnIX1Y}1Y48jVW#XC9k5~4WTVsg4_-I|8CnYWYA z?#P+%K_*m3Uo6H6yp*#b4ho?K#$gL?;Fp|*X;BG1Fb_u%B4<%_GQ zpd7kj7X0y8&XP#*Mm-F{DxAj$IZG3xB-&#tcH^F$WfroaCi-J3PT{qjS8cf;vC+~*_sF?&<<0u3wPyg^Fn6SKtC+ONxYJ?JuV8PCB|bbZsNC`9qCXRJux2v z2$i!l2J)c^MqvZ4;+vdZDN!C>F&hW)M9%KW$cg$Giq*J)k8<`TK`C^=H0;5BIeYC? zueWpwZ?iWmpVh(uEW>HMk+Uy8ilQ|pVmof(ubllEP!+wg5XbObj(;o^Kr@WNCS1o4 zIR{ds0=i)i4&j-cgHe$i4KW;RaS5O097={V=!6;Ahlg?wM?iMe!62-_S-g{TBq55U zEhb|p?#K!7AQP&iFBanjUdlNd2ZhiA?>k#jsc@}eoKuNWveVmlrNY~sYR_j=u^ac~oVJh!HPIhS zaSE^HoQa1bXoU&bh9LZrb2dGypcfY4D8l5Ni;4VbiqY7JYxpkbd@A_B7k)U1r*bYt zK`u1FFs#8ve3El9DN3UwreiN2$hqVo8){=9mg5ZG%DJ2X#n1+mumiWn$Sq_=8~qA*%wJhtK{ ze#^O?4wcap^AUhhId@_pADUnkHsC70$+?>n<4BysE?sojSKiF=YA5D zLI+I49^9Anz(!Wo!T>D8X}pp1Fg}W+H6~&^ZsD(-M;TBRy|EC-@LbO0SSWyI7=ul? zjvsQKq(%jF!yFvKGdWMAA~zahIM(74KFfKQ3}w&>Gq5lEoQ?Iu+dO+HCpZGKqYegP z1)Pqc@j=oro6L={nEDj2x1;$|uZs3=k=V?(1JunYP5F+PA zbmT>2jKq3e!B;siQ=lBWU>5xGSk9|R@J2lh!77}`2RW}3qa@m6Dt65yGgL~MR?5*_D#p8=DtyOo5?AV6T^cw^ns8d0Z%OM) zNP7oKr}j!WK1#vf(x=YySZn2p!(CiyQp%Hy`^QSom8bf3d75*dmv3cxhK!SE@^N_< z{*`;5sDD3s&hC`w)?0bPvdZ(dz1;h&;y;voA5UK9ES8xiuUgELY{lHd9zOCK&aB31 zN947HnUq_YeR!ysye=^B@$O4`nYorfTFaf&m*}pnObKNb;5=FRX|l|`p0*KX^)4f8 z6lc=RJOy)Z-<%H*=qAgYcYP2l%befioS1ta$((yJ=PS%UezRZP>`ONLU(G&Cv!BoG z8(YOYMzasb?6)xQ_09WZ-c!2o!_0dZ^L*btM>fwZ&2usH{KfpvH@_#%?-uj@*nAH& z|K6K_SIpn9XAjR@X`Sq!HrJ$BF=$hXHe+dXj5dF0Q@=7X8nDdNgM92i_0BpY0A@vd*MnRrp*u9aL<}=Pug&|+~gN>@88|XU2XnLXmdyI z{fJ?;Xfu^I%;-!Ifi|V(-p5nMpEk_-Y{fju9=^0;hUT;oxpT(1Fq87o0@^T#^Ija< zRHMyA+He-p%;_`p;0kcgulyd`aGtSk4%&304QB;sK9M_5VLNB<0_M<$^M>ZUt2tN6 z`91fXm^qJR&b^rP6=om5*)MMPC7b=PW}l_m&u8|Hnf*y-AB@>=VczTW{@8s_Y2JsW z;P0GyzHgo*o9C70xtMwWVt(hF-;?Hdi}`+RzK5BA@6Ep}!4WQ{3_bWyoA0#AM;kT^ z=h#e}=Tg=5v}sA3#kAoW!o>KrsY#nDv^guCc4))*iQpl$*-snp6pYK=b!ocNhWq+T z2Fp_;HEp>6&UZd-xbtsHY}!<1-EfDV|4F&`iQX>8x*0&5-LzqLYJ%*v=|~&ql9qWa z_uiLQ%=zr$M;m6FPLEETimV%Eg&sOe8|I_lD@2<5Ce0k|zbPJzHdScDGq=tsB>seHVo}=k zr_C?6`UxGTcnNAz##g>gko3gB%;j}qO8)nW<&r6%`v|)y;nKOPd z9c^0BW)W>}$(=WvxCU(|(}r`gJ~nMivTg>`W*==hTWjXRtmr};&Z-7HlXWE(ZJN+# z9&G~Uo~tzH_slsla~_F5C+_(Qvyb2G7az+W0<-_s?6Wlc`Hnx5yFbb7gJHjg`(EF? zKQ`|v&HJ!o0ssH`zIl#po>!XZV&?ga`JHcmPnzE?=KHbv9u{1ue2m=9{%PYwn^Ckm zLYtqoDL|WEwAo6Vm$b=9o7S{hN}IdVzJ#=?O`B=7IWK*VNSiXW;jYIt2Waz^HhE~% zjW*mD=^HA~(6qE^PMd|a36gs+^114?nM9k@GWS-x_ai14M4P>|VUBNEZ`yRG&05+# zm3zO)^v1NAOB-hE9*#+y%Cs3ro8z?kOPeCBn|`#}Nt?H{$x55{v{^};hjQmA&8knE zS+uz9xW}_w4BAwp%~;wTqs<@M6lUG@rOgi7yrE4N+O*?u+6vk{ zkbB=-*?P2@NgL)B_l!cD@~oSYv^h*0W*#5TPn({!*+QEaGINvN`Mimj(B_WZxt2b) zXfu^I=j6^E8_iktSwm^#Pn$2a$<4a)rOgJ~gvgu^ch4c4^RDJxr8&Q6&WV}xNaoy& zIbUJ+@tghPW?!<||9T<^Z936r4Q-yV=Y{W-X1|4buW#NToA;FFeVBRgVxI4t=g8)H zrFkx9p1+vi`R4be`P~w{!de>0fAjzM%tjtR+FX@VMW;fy`>iHzrqJfB-1`ypm15ltq0N5UFh{y)F4}aZ&3f7d%e}|qa8ugMr_Bwy zbH;yGrOgD^%}Kd)V9mTyp8>SlO`G?$$&#Hy)Rpt5q@|yZ31ZXi#7#WH@#`IjW(}nlZkc1+y$Ryw7Dm%T_W1lq0MyKT#!58 zYDZbt&2ZWrq|G1fk}Hj8L;OYS)_a~{c@dokxL%s&3zC0RFv zX|sRuc)toj9X%i%Oo^|Hxv|%2G&uO{y zM%$I3%^=$BrOij$c(ZOg(`GGgp30pIW9ESpEf;dvxPP9uF1V$_&j(y<~Xj7VXGn6*|teY>i$xRzy+H9arh}`%3 z=KZmGPifwVnfET{`M!CMY@Sz|=VBqn_XmyS|EK?3H(q=%h~1SoEHLk2sYGhpG^Nda z+T4%^$EHnH+DxF$Nol`_HpOT&fHu2n^PV=@Y15H5t7-FCp3=!_(~vfPw7DwJu;{d@ zNSiUVIZB(~v?)ZJKD60Ro7c3-%(`hyo8`2*FRyZmX;YUrGiY;B?tO1F%CT-n(B=?r zzSAZj>!t^7Hq++0+`0ToTGD1QZI~ZZAwF$t(q;;6&dQx@IlC0=W(aNe)8;d6a@}=JOy)Z-<%IO=a9{LS97k?oZmC&#LRglbMEEDd)j2DO-I_Srp;sa zR3@iQL)!S!=BnI%FlN7nd9QEYADj1-=6#rX?_!?so9D>pd1Xl38Epzh`=~+w4u#1+T4@AB%)0n+Dxa-1$pvBqD@)a z45!UO+I*u;UfOi0%_iD}$#XCrZCcP~5p8bC-0$w*Bc50pT0kmQNvN^YJ&WCs0Mw?f($wZqrv{^=*dvedc znDZ58A3yuW-F?Ys|7)t@wBh-g**9kPCz*XPX1|4buW#NTFK9uVMYOrabNqO;sX?2` z`R@7d_@~V<+8m(GSK8#EO*h(Xq)n*QH7#wL(`F%Uf}{;`X;YmxlW22V3bklcf;NL_ zvzIm>Y2!_s&a_!eo2T-0Pf44`w3$nr>+)=lNt?>F8AqGrwE0V$BDCqpy4gvax3tO1 zx@k|Fm9%*%_uk5p^=UJUHkakzD|W<(Hlt{Bgf>5EQ-C(T_?xzsHZN(Dk#*CWHcM%9 zSMGe?F|}zkjW*}y&QCm6hIKQHHV0_)l{R^3(~W;`Hqs_k?wpIt&1ticHbHXFA)E8A z=3M2P({j&=oh-q+8AO}CwE0LIZ`Ms`+N`C`Q}*tqq)lVm%%#nB_Ep5BO=a4Qqs?*J z{H0A1+VrE%&e`YgPXDLP8rnROQm3FzBihWN%{8f64BAwp%~;wTqs<@M6sApI+R!%W z4Q;Z}rX6io(B^?W>5|Z<9&KjQ=8`-uqtK>2ZAQ}OFl~O&CO>U@(q;>7UdZzz18rK- zW(jTX$h}8Ae=XWfrOi3H_q_Hh&AJ&%8-Ln-p-pbqjW2CB&?ZFgeLTOK(PjZ{Zpytc zp-?s2Or*^zS?#@OQ=D}(kT!d0^MN)wST~($vxYWLXecVpA0Ds3jv=A^t5d1zCNbu)lAyJ_>DHrZJ>9cijF~^u=(*f^3IkfwW-)DU%RTRE&Q+T8dx2-=o=4)`i+jGp?Bh54#m&BC zv;WoXvuqm79v!YE=+VrQ*F50}KO*Yzepv@}UJd&qYGTJnt&1~9S zk!M;o+Ek#;XxaqO<`-=W(xx|Uw$bJlZ8FiO4Q-au=APX9s@v3|&2-vakb9r#^0KU( z;j}qOn{Twq%ev`Kn@zL{lY8$=mKL;GM4MZ3=km9&L7U06IU{#&@~V=oo58f%N1IQy z$;rCuLYsB8c_uqWD%v!m%{!^ zO?}$TqRnNgWK`Pt&}I~Ej?m^OZ3@t)7j3rE<|S=1(xx?SmeS^~JP{JorZ#P+(dN87 zWg^n13~h$d<^XNJ(k2gWy3uAMZ9?U_kd`*hX|s?vLGp?mmp0XDGl@2*t+~j4$$T+ZSv5j8*MhyCX~HMX=&4(HVbJJ#Czhn zw5k3lOO?U@v?)!Sp|tU*%@^9_rj0LcHqa(S8j^-K&1kcLHa8{zIJBuon~AhJC4Kdx zO>x=`q|F}Me4tGZ+H|7L8rnROXLt(QG@{KM+FX<8Pz>5sqRm*^9HY%2+7zZuU)t=T z%^TWep-ns5tf0*Ux%VQ^tVf%fw7Der-pb46SvMnTbC@BrCB#aY2#0uFSNx=`q|F}Me4tGZ+H|7L8rnROyI|(B>#@e$%E9ZTiq= zJ8fRmCNpi?(q=hr?#q)mF>UJ7W(I98%F`nt48x*0;7{j~W^n_R4$uC!TCn_!vy-rRe|lFp~i z4Y~L6RIJLnnLwM9a_>tRTa0xxfHu2n^PV=@SvMVNvzj)K<<7N?*^oAVw7DvG?pT$I zteY{kIZB(~v?;{8=|h|Cw0TXN%(USdn>p{wxk~r^o;fF$dIoJS$~|9U_VG6xL7PLg z`A(aBwCO>c&9r$gcOQ(|ZxOn<#rEsD{%ON^wm280G?8dimNvs_bC5RQXp@&V-D$Il zHeu4nbhP1n#Mwo(xg~|gqfHIkOs35ldD7doDM_2bwAn|SPqfKNn=Z6jN1JEzY)M6% zCbXGHn?QM9#-dFX+Ki{o33+7_ZHm&SKW%o=<{fRav2HrhW)*E7$-RF!paE@W)8>lY z`w@Rvpv`F3O#p3v(WW5lrZ;W2(dHFxGO=#j&}JEJ?#Z1qe!LECrqkwv%p6#E-e}C> zv^hwdZ?wtFy6H}vO|%Jn22*`cIp^wE0LIZ`yRG&05+# zm71rdO=H^3rOkC|QB2xYrp-9o9H-4++7zKpKicf1&0E@JrA>R3)X!Dac1!&WYHd|@)vU#xr2mWdEK#H4$ zHuY#TlQx&68c}Fdo;D+CbC@x8EDgrHcM!8N1g-;Xj6+eQ)zQf zp4t&;Q<^qIY2#0uFSN-`8(-RNpiPK8_tMa&8EqEO=BB(7$DvI%+DxR)DS6fNqD^ty z45ZB-+I*l*4%SU4+N`0?6S?=izG+08IkdSZ_a45il~^}pX>*J=e`r&fb<>wNJ81KU zHd$CV?P#-tHV@>^vwmNXHZy5+N$$K+GuN`%NZK5x%@5k-XWjIq%@*3ckUJN~%pW(s*vl7#$E^TJe=Azttksp?0-Hf2kA=-SWO+MC5587;|&2zc; zS9NGfo5i%bE%$zr)ir4|g*Ip9&VPSWigh!DHv4JwnKrptH(hD7o;EyVYm%BaO=&Zq zHaFzXQ8M#-%v_jLJY&1*p-nN?%>df$X5GA}O?KLJq|IvDJeGSN$((yJ=PM>(WnXx7 z+Ek>?7}^}A4bRxZ3KbgDmG5HzwaG@C4zyWCn@3WqWVC5Oo7uFvA`Oj3n+miUO`8DP z{Gv@k+VrN)Hrl+RO(xp3q0KVd+>@tlBHGlU&2-vakY_|B+LWcuaM~QC%{SWQrA>F* zY@$t=yrQI|O$*vAqRlON`NX454cbhm%^7)(wrNw6HiK!ik2arZlaqDRg*NMG^GsIs zRJ3VAn|ZVelzZP>!z!$s@w7Q1i$BNiy)Uc#(`FZK-q9u->!t&3R?+5>+_}jq8_;Gp zZLY|jYuTg%ZAQ~3fHuEqQ;>Djn>O2M^NKc^Xw!x^%V=|tv*3wnQ-?OwX>&pD`8{(^ z%y&3#4$|fuZSvBlJ8d@6CQR-=%di$Rwk+TCPn+7bnMRxQQtpVfDMOoKwBhfb?^oL7 zp-ng1Y@|)7bRaEln$u<>ZGxn4acNVXHj`*`TAq9sZA#E)5N-C-<|A#qY15fDYiaXT zp2I0=)0j4MX>(nkpD}4unKt8SbDTDRX;XwY{b;k3Hg9Q@l{W2ZvywIs<=*dpr9N$D z(dM$u{k!fxugONy<_K+m(xw3GrWb9t(&i;?GSa3sf76!I=C0g(EUwq4%{1Dampf2ZAQ}OFl~O&CO>U@(q;>7UdXFp2HLcu%@W$&kyoDtw5dg#skAvK zuN@I+Q<`-%ls5jf`9hoAwDD!#Y@khu+Q- zGmtiWX!C(KIaoKHXtRbkPvp+$t=fn-b7*r-?)*eEr*HmP+8m?JAKDaV-Snl+4%)n- zO%~d;qs>X>*u1KWLMmb<>kJTPl6po%EkJ%V~38 zN}HHAb!jt$HW#H9k!e$oHX~?rh&JD8laDq%XtS9%&!y+-Y15K6i)nLPo(%D6Q@ZtoO?Ntnl?>oGoLm$ zvetRaceDTgy*VSrvuRV3HiK!ik2arZlan@GXtRzs&!okvXw!r?^Jo(&-HAn;Dzq6- zn-lURjUlZ`eVXtRnok2u?!j5ZBuGn+P7 zICm6{HWg?ynl=Hn`L%g>(f0qec}ttDwBfVdD{1pks>!-)QJ*%mXmeSb8kIIav>8R4 zBeeNRn*y}yMVqa(c}bg$v}sM7rL?&#Po0FcsZE<{v^g)&jEJ-;Lz`i=IY67Qw8=x8 zZnW7*n^1WrNlTmNv{^`-AbHh~OPlJnnM9k@@|tbYrUY#U(Pl4gKGMdUHl1m+mNrl2 z-dCNiF>U73=DOVbL_1by-HfBnaoYT)O%c{jKicf1&0E@JW!@cQxlK7k#Bo9@=!H%|_aUihcZkiIpb2jgx>jwP-VyHs_?$5olAIHbZISPn$2a z;d|X;U)pS-O^9@!|M@%3XtRJeH|2>Ghc?w{Gm$o@%vvNqSm~HdASH zPVPMx=Dq~)p|tU*%@^9_X5IMGW&>?Pnz&34+nrcGwrw582*+T53C zWMbOXrOgc5T$JZXWZIOY%?R2YqRn^Om>3<=m>Qn+dczDfeEn zz+$YM0kqjooA=EO-8U(}uM1qs>*h^Lfo2*!5#*bCfo}X;X-G(}yV z(}p(7Xmd}>lZZBTXfvHQ7o_fyXj7Ip!)bGnHs5HImp0vLvxzog()V<HnBGnh8}X!D6SIcd{{HtT5fOrBq&qeeMOt1wCPWqU9@>en{2e{K$}&xc_gpD$!ODnHnVASMeaS~l`F7rc*fQ-fHuEq zQ;>Djn>O2M^NKc^ST{Uld%BD^_vGHk6T1#=rqkwv-1`!$m1W%wr_Djye4|ZX)=hWX zY@$t=+_{!PEoifdHn*5B7LPVHXfv5MXXMU>sacYBGnh8}X!D6SIcd{{HtT5fOz!zT zb588}yjNFG@ZbJ_Z6?v?w3N=GO$pizqRn2~e58#xZ93CtEp47kn^V%JF>U73=DPGE zCT%LyW*lvf)8;R2iqNJXZFbV;Ep4*Wraf&|(&nK&+mh0zK5b^v=CVAmqteEQHlt{B zgf>5EQ-C(TXtR|zFKLsJHmzy1ls0$ewKE}YYSU&KZO+Tx~ zbfe8i+Jwrze|LOy+AO3^klgzb|6gPG9ar=J|9|}P8pqyy?>*bY-h1zzWbcu^QXz!w z5JCtcBq3yn5JCtcgb+eT{BE7!|M*et>3 zlBDmiGVV#rI}@9u*nGyOHn|yv%`R-7i*e5QlD^oi!{)Xa=fI}V8%>{U`SKh#f3T^G z%~Wg-WAg!<>f~k^HrugzBF1?N>F?W5e}8!T`;f!4W7Cq{cwlqdael;?3^s?bd5=vs zY=&a94IA#`oQq-81DloDTo>7#4VxC&xMOn)n;+QNVKWh%{n)$_rE(=~24S-qn+Ku{ zE`&`NY?fhjMU<_XuxWzL9Bhtb^A#H_Y{p`<2b-6oRx5|i0Bkm3b63=1`LOAL&0=gW ziW;C|(*TSqhsz*sR6omKgU{Pj5qR7GQH0o8Q>fAvaU7 zIfTu7Y^q^1lrwD`Hjl-)_oZPEY*u1(U5s=2XSE}Z=>GKnf zbNUKw#^wRDtO{Y%1)F8qToL2^k@Pti%jRHn9GkD$SYb03n?2aPY;h$M*Y>};xi3<% zAU2(`S&GeNkuC;o8e=mXn`78ux!Dq%G1%2C_BiMYzrY1Hcu-SplQ&Aq5#HKekYp}T~YVn-dw8myWHfOL&!^RPt$=Dpk z<{dUwu^EERR%{-L`m`uE-LYAL%{9?VWyPjBHuJDKiOqLxY{|_8Z1!RET8w*ML!GhN zgw1_1?&15~8JnfpTo&V=B#Xx6W;Qm*u=#?GCAk@c&2DU7h;g3vvHsYs$L5Y0=Z!Mg z(l|e{zy;>MDcID#2=n;qCZR~etn-FZ0v8jR0aBKpwi5FR30-Ijgti~of^WW3wj34vFCIXvZ*f@|IS8Reg z(~_{Mg3VxT{IQ7>;~b^*c|GZKVbbT1Y|esBGi=8lfL(5%v@|vVDk+dYjQIVo4we)664(D^!dDBH)3;- z`IY&x>4?n|Y%Ym$?wE5!ax)W~qu6}LrZzUCu-S#pb1}X@{7v6%g-%w_VABDc#n@aF zu~D&UfXxhSLb3UTO)YFjVzU#QXCfy{Vbcejwb5oHrOn{<}5b9v8jX26l@M* z^B$XO*bK#H8#a$ci719m4{TOqb6u3QY}mBG#vPke*!;l84x5SC?8oMfs0%7#GYFf_ z*gO#RTp?__V6zOHE22pzY?@#*2b<&Ae8t8Jo3YsJ!RDnH_q+QJz-9wBcg47W_f7|F z7GrZ!jQbJuH6S-LunEQH6E?NT%}8u^V)IOldn`8g!DcNsx5PMSJbj*Z`rPENXR-N> zO&x5eU~>qY_t;b;H$$=6hRtI!&g)5^3zI&7B>cJ<=P9JWZ~u+E|HIMT6aRO;$ut?8 zgV?;orYbf=u-S^uBasP3vFVP@3T!wB_h-eXIX3gKIf>18Y;3WafXzN^UW;O15gTW0 zHeqvL6xV{-bjD^WHkU;?Xuzg1HnXughRqjjEU_7b&2DU7h+3yCHvO?#kIfxXr{=|` zJvNK5xgctYf=zvFrekvin~&Jk#AXCGJFs~wTK$sP^u}flHaA6^krSKNJ7$4%LkHZhVu&)PUQ zIek8Fh4ai66>RE}n`zjDV3UkZ4Qz&E6M#)TGhj+!(+iu`*hB~R9@-^?%{grTU{e>H zsn{IG<^wjIdlQCXvmKi!B3p`M(-WIj*xV3_%Z^P;Y&@_zjm=MN?6H}I%>iuQVpAC# z7i_j*^H7v+g|X?1&2nt6it;2gHchdai_HmazF}jH%{XlKV)IJW;pMS$!e%2j_e9;1 zADfQYEWzfIsLwQP8e%gOo1@r##-=tlqp;b9&2!O4mBywoHtVptEm~l1Y}%5Wh1i_K z<_|V?v6)J44rB8Ho9g6d7&hCnc_POBRkM3yvkIFVV%#rsyd^ds*qp}ZCpPxjOd>Z2 zuz8D3Wo%rq*@DeOG0y2rp9hydcPxFr)&9)bG{t5vHYc$8R;J<-e%}A)=8Z_9O4tm- zW-~SqM7kEj2EPf*u(=}Q!_C~5CfLlu<~TNAv9ZEtEH-o9m+W%Z5!0Y}~Oqh0PCa?68@L&3*K$Gwax(*)P;5S7Qwy7s*zCmSnJ|Ckn_ow+E&thM&WlZZ zY!+d2L8O_2O?_;pV{-(XkJ!}2W&}1nuz4zStt2+Ru~~!7O;NJt#HKYi^RYRDO&T_i z*i6ReAU5x?sfx`IY_?+aNR;TJ*mTEc1vb}2&6yRO=Ge@`<|H=Xv9ZNw0yg`wc`fSN zir6?~vk9B~qTViuO=oPDVslxvJO*qUV>26@W7vGb#uA$`*zCsUg=p)`V$&a+_1N4I z<9_!S?a9p|Y%Yj#|8BYZ>G z#KsSsSW!F+VABa3Z)_q(Iiq9K2%A~hgkh6{jRiKNu?fT`LDV*7u<3`5FE%lvF3f{X zJ8ZnLIWOvY!KNNI)36D_CK;O=*bK)e0GoKx+Lyql7dET0i56{f4s2Rs5VO(y1FZKG;Nwaj%$PGjijGO*l5G*w~Pp@z?}mlgO<1 z3fK(9#t)lVG0x{5)Cn7JY$C-tKXFSVax)8?Fl4?n|Y%YmR)UauY%}i{LV)GfB+SrW3W*0WkMShmXrY|<@u(>UYLvC!^VzUsNbJ+aB zrY<&9u{n&*2W+ZiGYp&U*gO&CcX4cbVzUaH8=}_Dj!jE!Jg_;9%};FXv6+O;0c_r4 zQyCi|X9W*jzqv3VuhrSjM~VY3mN zdt%%pp1&hDOR%{l#yzi{8j_ot*c`>?Gd8u!%_wYkVe?#!`*`m6#bzBgx7mx38=JP+ zEX3v$URhS_~F!sb0T)vy_g4LRHLSmZ!4Y zJ(3Tb4%jTl=Ax*dRBRexGXtAYY(8OA3!9PH?8N4oXfsP;(+8Wi*xV8=EEhIyuvvi3 zS!{k|Q-|D4!R8P)@3E;yZiZsB4V%Ye+>f}l2R19Qxh}^2RaaV&8+U9@Vew$5;8);ky3UW*;`MMJiRq z#u=MU*xVO!DTqyHY?fkkS;XIfO=E0kV{;6fFW6XOGX|U8*t`&>YFTXhW3wKcJE9ED zi%olM7GZNilE*SY+7S8ADc7S zq+#QT&17s2V)G80s@M#{W-B(2M5|pCo9@`Gz~-80qqAbu9GiLAoW$lkHn!MIz-Avd zuSNS(5gTW0HeqvLjQgq^cgAKZHkZY?Pc(gR<t50sGV{-(XkJ!}2W&}1nuz7mr_FV3h|JuZe6wZT9 zJ8ZnL;UMSoA};l?nTAaWHp$r3z-Bl$0ocTgJSu@rFKkw06D>-y9N4tN#uJ+eY<^+m zfQ>6Q!Pq2WQw5vB*!W`;CrW$~Y`S6NgH4pEC9`1D3>!CW!m&xk#s-`5*aTsdDC*7% z*bKzR51Uv~pBKQU6E@!1M2c2M$EFcBv#<%nCIuS{Y(`@fh)sfMyUSqH4;x=>VnlnH z2b*@-cwuv1^zwpDJ#sS*n-FZ0v8h3BhGP?eO}rTQ@SW&|&1!6-#kePleLTj!FNGqo z`Gt)GHm=wNW0QnU6>>8e8-Hx#m@Qfan{L?nU=t4V#A8%*5s>HlMMnjm;=*c46~e)IO!L>5I)eY;KFXE;lx9u~~@C zIc)x5Qx}`5*c`^@12)yM8HUYvY@UeLzc@BMu~~)94be7a$EGDV9@w15<|j7x*i6Fa z05)&2sf>*aHe0ZHD8{|WZ5Nx($^GJ;QcQ=T}#yu7;&e&|i=Drx`jHl1DPM@2cKA-ox0h`9;W_HZ2A0;!`jK?Mjn?w=23fK(9 z#t)lVkx2!x>4c3pHjyI1IyQ~4nT1UlHYwOxU^5zz@`^AtFeg|b$SkLT4CdfO$0W-uyMe~ z6`NpelCY_Q&0uW&v56C{VG(S)VdH~MlxVZEVABj6H*CVONyWwnoAKBLVUsAjp#nAo zvGKzuR`ezXu<3-2H#U)C+*iG<5xJR#O&B&Q*jSL8(bxoHlOTpHWw7aojW0GaV%+~S*j_e@)d&25p&xv^=B%|dL>VeZ zo66X@V6z3AhoU4G#-=Ma%dxpCYR$~pG{t5vHYc$8hK)5gkQ ziTXJ|HXX59g3Tq-EHrEyVlxw)qu6}LrZzUCu-S#pbJ32K#-=Ye>#(^k+SlCJw8ds2 zHs`SUgH2s*rebp#n-ADjCpW{e*^bQAVwjX0SPh%@1tsu$hR>er(=|46cOD zAZ#{c^FU;4A#A!}vkaRnB9Aj+(*&D2*c`{^D>hcxjKyXTHZMgPRt}p1*lfV&t|$Td zu<3x!Vr(vo@>Ip90X8$R3B~3UHnp%BiOo)Io{2iL6gGXZS&PjrQFrCSrVTa=usMs( zZ*1ydGXYUb3sI95w^6 z*?`SmG42;}?|{u>Y%Yp%{`=_$8sKQ`;Jxg*j&FE;J5 zS%l36k(CNI^|6_b%@J%qVp9{F5!md&=BX&fOJdU-n>EAY+7S8ADc7Sq+#QT z&17s2V)G80s@M#{W-B(2L@iYmo9@`Gz~-8$eY0ZI9GiLAoW$lkHn!MIz-AvduSHF$ zh>bHgo3ObrTG@iwbjD^WHkU;kV8Es^HnXughRqjjEU_7b&2DU7i1w;1HvO?#kIfy? zE9AwdJvNK5xgff;f=zvFrekvin~&JkBsU|l*@4YdG42!n-W!`W*xY2VTTX0RV>2I{ zGuWhI<4A5MV{;Ijci2=VH$$-5ip?WoZt{ya1Niy>?@Y^@1DjUZcw!TQ%`a>muyMsE z7@H()s$er18-Hx#L~azprW-as*hGnvBMUaouyMmC9Gg^ZY_J)RO%OJTqO7Ta%|LAY zu!$ArRsn1}VdITWq^P-dY#L!R3!5-(Qn0bWW;8Z|*d&PRTLznc*!W@-BkG+z*tEmO z3!C$z|jRQ8W z*aTyfgiRG}24mxoO`I6_yYK6UjSn_aV%)!*zUTG38#dwCq+(-(&3J5rut^l-o}@_w zvGKzuRt#GUVABa3Z)_sPIA{D_Bj@`ixu*QJvBqW`HhZyoCDNiiHcr@V#O9ueM}BNN zVzUIBOCk{(HVv_viOo@LK4ViGn^D;8!sfXstxIFm7n^n1+!kd)Zfx3Ovk;qe*!;n! zE;duKIgHH*Y^q~344duPJQ1~Bacp{GvkIFVqAtpgO-pP%usMy*Pi*Y5nS{*&Y~Es1 z8559#AY_5vtof(^^*v!S|1UBEWvBqW`HhZyoC3?Z~*f?Rc5u1CW zcgc@UM{Jg0b4m2&8a55d%}i{LV)GfB+T>;wHoLHSF2=o;mHT3|4x8Iz+$%P?ExB2U z%{grTU{e>Hsn{IG<^wj>$<45CKF(y~uT39p)?#x@#6A}`ZLnE@%~@=IV^as4DcBss z<~=smuo;TYHf$b?q!q)a2R19Qxh_hbY}mBG#vPke*!;l84x5SC?8oMfDCa6+GYFf_ z*gOzb3SrX)n`PKs5w(6MY?@#*2b<&Ae8t8Jo3YsJ!RDo?7t3KY0Gkci+!c+`X!$x| zvlyF;qBT;nX@JcPY(la5giS4MMq;xQn`fe3E`?1WY}R6POY}^+uxW$M0&LD=^BbEw z*i6CZ5H|0zsfNu^Y_?(ZSd4qbANIgzB{tW^xaT!}pJ-)wY))bG0~Wu>3mY{+0U51W(Ne8Qfw}ZyfVX^+h!Y%YlM zQNgA@Hq)^=g3U*4YGN}2n;qCZ6?H~QYEHh09hziMTB zaJU^5UKKWt(}vn+s3 zCv3d2i4<+Dj!h$MW?>VCO$s&^*o?*|5Ss+izLmkIA2z<&#E5Q_2b*@-cwuv1bbe%Y zw|dx2!zKiqWNd0+GaQ=$Y~m$-U$t>Ba*@^8M6)+E2R5z9jVCq{*!;rAf!w%a6O2s~ zHdV0UIkw0C*u-^O|C;mZ@0oT(q-b_*T4LjY&1r0YVq=dD_j4Wxuz8D3Wo%rq*@DeO zktc<*>59#AY_5t@A~QBkv6+j_32eS$V~x!?Z1!UFN|YVtv2ns?BR2O$d6plWj@T^0 z=8~wTHEbGUGZUMm*nGyOHa4TM*@ew>Q3Fe3(-)g{*xVNNMQ&``VzUsNbJ+aBrY<&9 zu{n&*2W+ZiGYp&U*gO$!Z*gpTVzUaH8=@s<$EGDV9@w15<|j7x*i6Fa05)&2sf>*a zHe0ZHDEj`w*mT8aIW||txZnLpQ*7pvn-kc4!^Rq$aoFs|=9L)t@Oe04vk{wnV%(Dy z(Gi;^*jy6!SQM*KwpIq4uh>{&GZvdY*t`_!Q4X5{*l_lF+!a}s51S6yEXL-dNVJMg z18ins6N=3zY-(XM5}TdaJQJmNDQx;+vlg3MqO8q@O&e?$U~?9m-`Lc_W(qckuz8P7 zHEf1rvkjZaqV_9>O%H5VVsl;8_1Um#fsH#hr?B~fjU6@yrX z3O0wZd5=xCmIWqrR{fWoY$dVjjm;WtZi=+biA`&4=3{dPn>1`3v6+m`L2TY(Qx%&b z*lfk-ktjKfV$&U)71&%8rA=0BnqxB$o0Hgl$Ho?$3E1qz=CvrdD`Mk}%_eN_i<+k( zHl49qip^zF+Z(WHjLmFpj$!i!8%u1)V6z*W7oy%Ri%oxQ)?;%=wETIoX^+h!Y%Yk_ zNx`N*Hq)^=g3U*4YGN}2n;qCZ743dWY2ET^u5R}PGa*N8(VTS0h@i;ycWZ@ir6?~ zvk9B~V%*2`zVoIz%OrzMGi=02u?fV6=h%9b!KNQJzSzWwaX;d=cG!4f!*guWf=xYg zGi~JHu_rRvjKXFYHqS*IN@LR(n|0XS7MYwIo3_|2#O53}f3T^G%~Wg-WAg!<>evj! zW;-@dM5$XGo1WOL!sdo3)3RgJ5*rU}PGj>E8+&XfVRHbRx7bw1#s!-#*gO=qL1ApV zVzV5ZtD?@#j7?K)=3;XKn{U`yV>1q$z1X}GjTgkU{7%?x#O9u8jq_vE5t}90ToP@L zhD}3kW@2*`o6p$P#%2^YyRdmKdgjvD^u=Z!Hn&A@mK&S4*et~695#Qjsf*22Yz|}d z0h{XB48vwSHc!NmtvEJ4u~~)94KePk?$Ht(4{T0j^Aj6;ax)2=1K7O9rZP4z*lhV| zyMJN^o0ZsH7pa;Jn-@%^++xWAi}tQ-!eU zg3U5)u896K6E;n-nS;%7Y`$V+MQ+Aovj>}(V%+n(Y5+DHu(>P7J$%s}uvv`FMUU8A zIQ+fd9K+@dHkR0o!Dcr$FGPlx#il#?~bvOO<0?Xg*e%>|Ko4x$hBv6+s|5o|tU zQxlsJ*zCaOsVE~#V$&O&HQ3w~WoJ%oT4OUGn={y?VdIF+WNZ#%^A4M;*bKpDD>jcr z9bFWg?%1rr=9;LxvtrX6n|aur#O6CTw%AO-W*;`MMYF1ijWafzu(>bVxPsVp#%3ut zmqiOQVAB|z+1MPz<_k8K*o?tuH#RRsw=IiJe{9xcb4T=vd9i7a%_3|rh<-r9ram^) zu{naxM{H_hGXk3(*gO@3LrH9UW3vXEn_}FH?Ae;!%*W;oHfh*6lAFoc9K_~bo9*Rl zWw41CDOLiTUf8V0hQqIC4s2Rs!CW!m&xk#s-`5*aTsdD9ZB+*bKzR51Uv~%M`$-6E@!1M2gy9$EFcB zv#<%nCIuS{Y(`@fh)sg1FUw%l4;x=>Vni#S2b*@-cwuv1w1I+6J#4076M{`LHZ`yr zj!ghI@uD%OL`&_3&1!6-MX!_tn^xF(ViSSQFKisJam6MWnci4sHAEZ8)|#toZrY*MkYAvfc(3Bo2(jQd2-48+C{n^Ml;X2gv8#dP1jKgLxHm^ihm&e8ln~m7q6S#(^kN=$BS+G4X1n{(Lw!KN-YQ?WUW%?E6%V>1k!?btjK zbwhD%dSbH*n;W9u%Z^P;Y&@_zjm=MN?6H}I%>iuQVpAC#7i_j*^H8)+g|X?1&2nt6 ziuNEgHchdai_HmazF}jH%{XlKV)IILm-5&+VY3mNd!qa2$EG7TOR%{l`XddShS6!R z%}#8diL@$(O&@I5VslGmelBd!NsN!=?o`?%15d<_9)**i6J`KQ?bf&07hZLD+1@=7Fdk3SrX)n`PKs5p_u> zY?@#*2b<&Ae8t8Jo3YsJ!RDoC1-R`*qp-V2R3%tOvGkCHgCkZxAMcF5q(Q@E&prN9h(){ zToZB3icNED=3#RZoA21zVlx4oeb~GfIa?7MXKXfMb6@07L2No>vlN@lqSP~B(-@oC z*c`*=3pSS6jKO9%HZMfEP!^m1*sRCqj;Ly0Y}#Y92%8I{HdL^wkIi&!j$rcjcryIK^R z?%1rr=9=hPvSQO5n|aur#O6CTw%AO-W*;`MMPE@78)s}bVRK*f>jkmtjLlMPE{h?% z0h`9y%*N&zHeax@#AXaOyRms8#y#TO`(v{nn>%9M^BUit+$=ibVbddn%`9xfut~wj z0-Mp;1Y(mQGNlYQ{jl-HCPw6N9&FlSci4yf# z7Hpbf0@!rI#v7YR(ZY3X8euaFn=our zu(7~qG&X_QB#7R;3^x6+@x>-aboV^iw8O>=oAbgkicLLirePC;O)@q$uo;d`05iz#LmxPvmKi!BGrmx(-WIj*xV2q zmK~dx*mz*WUt;@DZ0xa_gv|kL-eOZ38y9T0VDnIvnuW3Hip_Ftu8J};Gd4}JnTyQ{ zY`$S*jmNGkE!aF1{a0aZx?-~&o2z1Q%#2M_ zZ02Hf0-JBxSYtB|o4we)663z=n@-ql#O7Yd8uMFZu(={qJQFreu$hC+acsU~V};FF zZ1!ODQY4@pHUqHPfX!Wzr}?nyfX!lTE{al0#iju^Gq4H8<`Xuxuo;QXPHdiuva1v} zeXv=J%`H(9a$(a3n+4dM#pX9Qb+DO&%^_^wV^a;Aq1bH0=CP=Iieb|Oo0ZsH7xh&( zY+7LBj?F1-eqdvV%|vYWWAjF|@s+R{gw1Ab9*DND5H?+~S%%FO(cWajrU^E4usM#+ zS8S}X8H>#xY+i~!sT?)~u-SmkUD1Q{VbcMd#n@aF{hf+U18ins6N=3zY-(XM5}Tda zJQKr|QrPstW-T_i#JJ!6Y#Va30GqSe{C>0VGxxuLuQwmDsfo=9Y<6JtRHRo)Yc224drfO{^%n3t-a;8*gkPMQNvF(+Hbc*o0w|f{g_>qp=CZ zCP9=tWw7aojW0GaqUOtkO*?G7usJXMN)tBqu$hKU2sX*s)WBvqHUZegiyB)3n_k$g z#wJ>{f;q5hg^edR5!n2~#sM2wY=W^#!lnu~gR$|)CQh`6MX>3HjSn_aq8G`6O*3rV zunET|6&o9D#$ywNO`_;qD_}Da8$WDfMSolXn@-qxV-qQc;yN~su$hHT7&a-`SYR_6 zn?P(5#1K#hn||2%ViP0AJ>tpjLP}1H&S29Jo0-@g#pW|MwXqq6%`R-7i!3OOOzA`s9ZLwL1%{grTU{e>Hsn{IG<^wj>u^EQVc5I%Ap;vKidSbH*n;T+S zlO3Cu*mz)b8k?Wk*pr({_x?Cl%V0AUn{C)U7O7JV8}8wHt;FWK$kc4uaBW)Sj?F1- zeqdvV%|vYWWAjE7sf5iSY&K)_K$QB0u<3%$GHkAhG9wc zEjG7AYnlt2HrOn{<}5b9v8jX26l@M*^B$XO*bK#H8#a$c&sq$d9@wnJ=DO%DvSHH# z8+U9@VeZmRxP-)Uo%nmEEyd=tNOc1?jj@@H%`t4gU}K5R7;JW9^Fkz~ zEH?eIS&z*fk&k(?X^+h!Y%YjWOTnf-Hq)^=g3U*4YGN}2n;qCZ73FA2YlusnTO3u zY`$Y-i_HXV_F?l{G`EV_IAgO3oBN`jEQn2KY?fkkS+pMpY#L)T8=GU;e8I*Nn=#nz z#^#0S9%ZrVkIi~)?uZ_d7n}CjEW+l3=xGWz^|6_b%@J%qVp9{F5!md&=BXGKl*Fbt zHfylCDTZ@7v1yIXd~D9_y}y?{{IvbNM6~U$(Ha^%yi4vFvn`YR!VH1u`DmFIQ zjK?Mjn?zA9D_}Da8$WDfMIBoJn@-qxV-qRrULBi8*v!Hv44V{eEU+1kO&~T2qS=(e zrXM!G*u;o7ArCg~u<^p?yzmk!HubQXhD``I$=KAuW;iwh*u;x&Ujmz6*sR7TT6EVO z*tEjN6PpNZeqrN)jVm_6*d$?71)IUx_+t|%hB`&C>4uFDHc?`jngyF?*tlU6j!h~y zHrR~ECJ39v6YlM}#{ac>C{m&@HeIn1bx1K7O9rZP4z*lfY(p=fUlW78Fz<=9*my-H?mnqo5-n-kc4!^Rq$aoFs| z=9TCN%VXn&%|>kQiT*x6HXX59g3To{RM)U+h|NrFj$-o}o7&in!e$pX&&3c@8k@e@ zti$HE7(V8nT;NQl|JmeJv1x$K3~WNN`Gid^Y(`?U6PsruYfE9%2b;Cn+!DE+3!66B zEWqY0Hovi{gUu9d4q@{in`+n$#bz5ek40Hu44WRV65Rl;TvHk+||AgW&>Y`S2x44W&W-p_EjG7A z-QV)p`q)gz<_I<)v8jp82yAv>^Hj7#C9&y^%^GZOiqMhv3ZA0RcwY}vlW|1q8BZSO?PZoU~^6M9$B$zj?Fx5PGa*N z8(VB9V6zXK*P=hEh>bHgo3Obrh7tv_>5R=%Y%YtTw*i~R*v!V}7&c$9vBYK!HoLKT zA%DYSBhZM4I3Y9qD0G_1)FBrxM35HO)54v*o?;}2%ALF zd@5iw5F0;iVnw@F0Gm$Ocw-YOdNv)KM%c{4CJdVtY%H)DjZGjn38JqmgH1nde6fiU zJvt9I?XdB}=DhHfD>n78nTAaWHp$r3z-Bl$0ocTgVQmR)dSSB~n`klI&VfxUY&;Dv zKQo?bQ?WUW%?E6%V>1k!?btjKnN}Q|p4hCy=7z|T?AWx##sizv*!;xC9-B$n9Khx+ zHkGk)!Db6K4@H?-7@MxxEXU@mC}EkgX^PEUY))YF4I680#$mG;n^&SXE{}~9HXE_I zC+eL1*mT5Z2{xBRJ)vRK5Sy9U9L44{Hnp)Ch0QK(o{QG3G&X&)S%=MS(dOmGrY$xL zu{np$A8hJkGZmY|*nGgIIyS?w*^bQ<(OVYBrYAP5u(=_+XLf8_V&j3$X>5LCV~@=w zYz|=a7Msf0xL~sdn}=d(Qy81B*eu89su&h##-=GYbFn#r%{OeUu^ETWUia?Jxi|ZJ zrftUNfk=%)*mS{W88%l$Mr6XK2{v=EIgZU&Y^<;ui_IQvUW$Awhs^+NHehpCl-l{Q z>442*Y%Yp2TE(UTHZ!mZ#pV+>wXhk9%}#8diSnfsHhr*Ji_I-jt#V=02Ac)goWg3Tdp-eXe@o1xfj!{)K5sl~AAfz3*6u8U@y4VxC&xMOn)n;+QNVKWh%{n)$_ z?NlXf24S-qn+Kx(EQC!LY?fhjMRbQu*fhar4mQWJ`HGDdHe<2bgUw6P&y>Sv05%)2 zxhwkbeAslrW-&Gw#ZXtprU5oHunEQH6E?N58HvqKY@UhXd?{@DV6zsRTWwly`;@`v zf=EdPoBG&H$L0t&AF-*4%?NCEVDnUDXGv^&W3vXEn_ zn@!l<7d5CLHl49qip^zFUmLJ#jLmFpj$!i!8%u1)V6z*W7otrpi%oxQ)?;%=v;%pu zX^+h!Y%YkFq+nAYo9Wmb!R8}2HL)3i%?@myiaxm{HodV~gUwCR59P$BH8%6HIfG3a zHjda##^xY4@35(g%@Az1V)IB0(~4r#9h(){Toc2Qtk^WiW*#;tvH5;u(A0t%Y?834 zg3VxT{IQ7>=~D!oZrJ!>6D6`P3pUNLalwNa8euaFn=ouru(7~qG&X_QB#2tR3^x6+@x>-a)PZ@hX@`v$Hs?j% zB-qr$W*Rmj*d$|91DoO41Yi>{TBQ=$^ulH}HqoNFO9;IMJ&X!KNEFKG;NwJ~Rt9&9HI9CLEhoY;3R@k4+FZiK2g~fXzT`{IH1? zLyZF1bi&3Pn@BN?(6MQR%`9xfut~wj0-Mp;1Y(mQhEHYM7k|OI^xv75D?c_Ju~~x6 zC6TroHVv_viOo@LK4ViGn^D;8!sfY1OlfTTVzUmL+oI&njZIr@7GiS_n?Kmp#bzot zhq3v9O?7OBVY3~ZC!*Xdj!jQ&R$+5P)B@SDX^D*oHm9-qiH$urldw5}&0B0LW8;F& z7Hl4h`k*j2U9nk?%~jC~XU3)}HgmB#fz3B;tg#t~&0cI?iRND(8z*cwVslTlNBObo zh|LmgE{R@D!=@oNGqE{}&1Y1eyUD!MqeS2wa`eL&Vo7S8k$o5R?Az@|Dj!?4+o%@Z-~ERIc2Y*t}&Lk!QeW7E?0%xJ#MCnion?cxY#^!-2 zOA2Ar1)F8qToENQ6E;n-nS;%7Y`$V+h0R!O_F(f;)XwFw8Gy|OZ0?G>EFU%oJDxh2|)T-db1W&t*5vH6Wn9c-px za|oOF*i^%2C^p-$c`SO*V%YS+W+gV)MPHo_n-{&GgkBKGb4kIGd3I+HSUYlv zQy-h@*c`#;BQ`a$8G+3XY@Uj0D2YvPY}R0NQ`9Cov1yIXd~D8OlZK5WHj}YAh|N1} zs$w$)o2}S95-m$nY`SB!0-I~1HP4Dob8O~ea}t~H*w|t-0h@i;ycX?xMQoh0*@Vq~ z(X$uCrZYB6vAHaID+4x-v6+p{F>JnIV~NceY<6SwLiC$uvFVS^dTj29Ay-~(+GDc_ zn+sxSt6)3HjSn_aqI}JQO*3rVunET|6&o9D#$ywNO`@oCD_}Da8$WDfMGY^2O($%; zv56G*yN*pGY-V8-hD{1K7TAo&CJ>th(cH^m(+?Y8Y+^(^od=tC*mz-cUifutZ0cb% z4Vw^blCi0Q&2Vf2u!$FaehF-PVY3>WXwlE+z@`;8p4dcS^9vgXY+SJk#wH1yD%cFh z#vhwFF?bchrW-as*hGopLKbYAVdI8P_z(NfJYM;GrX9fMEjE?0alvK_HV;Kc7RIJ4 zHp{WODzYmxHchdai_HmazF}jH%{XlKV)IHAi}KhwVY3mNd!mfVk4;BxmSA&9lsy_Y z4Y8St%~5PVV^bTOQP}Ll=DDcWrLpOY%{pvui#k3xHf^z4h|M`{{$NuVo2l3w#^wVy z)v+0d&30^_h-OzDo1WOL!sdo(ld@ye5*rU}PGj>E8+&XfVRHbRx7bw1#s!-#*gO>7 zu`o7Wv00AIRnezp#-=GYbFn#r%{OeUu^ETWUTj{8{--=PPS|Y3=AIbp<;SKYHcPO% zB!=l4HVv_viOo@LK4ViGn^D;8!sdCqUF8;Ju(>NzDjzl-uvv`FMUlQLHVv@h?DGo6 z<`Xuxuo;QXPHdiuB$UFY4>oJDxg|>3T-db1W&t*5vH6Wn9c-pxa|oOF*i^%2C^p-$ zc`VASV%YS+W+gV)MXitxn-s2|6&ovT#$vMvo0p;mm&0ZNHXE?HE84q!*mS^VF*X-Pucl(t z0Gk=ugktjvn_Ae6#AYWp&qP053Y$LIti|S*=*hXTX@kuIY|diy8=E@VOu^<5Ht(^i zhRslHwqf&F457ua>4D8kY_5ypbGDaaWkp;2o zjLlMPE{kE80h`9y%*N&zHeax@ocLfW-}nD)a#z4+AU1y3#EP^lfK4ZCys?QCS)^mr z2%A~hgkh6{jRiKNu?fT`L6m%Du<3`5FE%lvbj*WIJ8ZnLIWG!-l(MlNHq)>P!6q4- z8rTfSCIFjwQ45y9rWZD=v56M7OAc&WVdIHS1UA30alpnEn_z5`u&IL0U~K%ci4(0z z5p23)a*hGrn zOUI@WHnXq^!zKkA3v5PX6NpWM=ugXF(+?Y8Y+}SvDi1d8u<^p?yzuK;*wn*j8a5%= zBx6$po8j05U=uHfgc8{F8s>eF`>ww>ZLwL1%{grTU{e>Hsn{IG<^wj>u^EQVc5I%A zTr7@FPi$6Ub3+s@J2ow`@xbOZHb1ek$7T{X2e5gIO=WCcu-SsmLs2dl#-=Ma%dxpC zYNpKCG{t5vHYc$8hK)5ggO>JyOVY3UH=c27FjZI%{)?ss7v>UmxX^YK5Y|dfx2b;RsOvUCfHXpF5j?FM^ zwqx@|^fkq?>50uMY;K5tD?2tVvGKs>G&Vo6vBzc-HV3eIi%n&0T(H@K%|kKx7RIJ4 zHp{WODuz3mv1y9U+^&nF$i`oriP-GN=8Z`GO4tm-W-~SqL}nDirVBR9u(={~G!r&W zu$hC+acsU~V};FFZ1!ODQj|vJuo-~O25jz%GCLnO9k5x9%|%g;tJpNaW(GE)*nGmK z7B(ZX*@?|FQJa>+rVlo2vAHFxTP|$cV6y<5v)KH`rVch!usMXydu*y!NvN!=?o`?%15d<_9)**i6J`KQ?bf&ru1RLD+1@=7H#~3t`g*n`PKs z5q&`>Y?@#*2b<&Ae8t8Jo3YsJ!RDnHa+kwq05%)2xhsZt`LOAL&0=gWieZt8O#^IZ zU=xbXCv0kAGZLGf5R=%Y%YstZ@{K8 zHnXughRqjjEU_7b&2DU7h!#;6oBr6W$L5Y`X?d|}kIf=%E{I-7!KOYo)3G^%%|~o% zVlx7p9oRe-{ai_GdSkN&o10>goY=I+WMX>3HjSn_aA}_LF z(+nFoY{Icg#l{Ak@z?}mlPJpA3fK(9#t)lVQT7(VrV}>a*hGqwsAJOzn_1X|VUvQ5 z1vaCx3B)Er)Cpy<>4%LkHZh{^&x1`nY`m~JFDkeH>W_NZOv5Gwn`CTiU^5(>0Bqt# zb1i{QFKkw06D``o9N4tN#uJ+eY<^+mfQ>6Q!Pq2WQw5vB*!W`;C;HSP*mT3j2b(C- zL$YAg3>!CW!m&xk#s-`5*aTsdD2DnKuo;MrA2zXKm{9@J}*Xvkd=W zvSt)@)Kwo2=QcK4-FK7ktrV%|7a~$(loA zl*yW-XS9i$V(Wj~L`k)^2)}ExW*K|mWX&oi&ScHnC*EYuCg{1znr-AulQp}9*CuQB zZtqOi9Q=|^)*Qn=o2V&vF<(uTR6C30?5wOW*%`(E@WX&pWyUCh$>Q0k2 zo7HhUB++aqRo#$pVy6pY8n$hJD9$-e7W9+VsUH?|JO4*aK>EA$B7C!sU z=&}qtXhxS+WQZAE)(J<==(2G;VMdp&-zhV??845N(PbZV-i$7X%i+7oBj=CZE^ml8C{m~F=ljG{kmsHm$mOhGrDX-9-Gl+8~xObF1w@zGrH`(63ysx z2zYBomt(~Hj9vc*vWfecvFYDHHWsNVW^`GuPBo*;D)^@vUDi>*&FHd8oad3@s5kMq zxAmNFMwgwxml<95;Y-Zua)@1KMwesCii}`LK*l{|2(N zh(BRQm*uZhW^`Hko;9P(I^=>GT{h8?W^~ymT{WZ2&g+I5UG@RD%;<86xMN0_V_a;; zu73mBr#{Ts^lu=0i`7re=&}rcW=5A))C)7ZtP>N>=(6!lGNa4Z|AQG_cHy7Q=(3Of zVn&xkN~#%Mjy}IKcKsX3A?Q!WrhfxDSVX#e{-0;PgJr^eGrFwY7Mamy?dNSqmrdAm zGrDYJR+`aem%PS|E_?6wW^_3O`kB$?cz#R9u73kL#&65m^lu zNRSy_*3k#d=(0&VWJZ^*SEw0Xb^*uC=(3LpH>1lT?z9=- literal 0 HcmV?d00001 diff --git a/pkg/block/indexheader/testdata/index_format_v2/meta.json b/pkg/block/indexheader/testdata/index_format_v2/meta.json new file mode 100644 index 0000000000..66bc623f74 --- /dev/null +++ b/pkg/block/indexheader/testdata/index_format_v2/meta.json @@ -0,0 +1,27 @@ +{ + "ulid": "01DRBP4RNVZ94135ZA6B10EMRR", + "minTime": 1570766415000, + "maxTime": 1570939215001, + "stats": { + "numSamples": 115210000, + "numSeries": 10000, + "numChunks": 990000 + }, + "compaction": { + "level": 1, + "sources": [ + "01DRBP4RNVZ94135ZA6B10EMRR" + ] + }, + "version": 1, + "thanos": { + "labels": { + "cluster": "one", + "dataset": "continuous" + }, + "downsample": { + "resolution": 0 + }, + "source": "blockgen" + } +} diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index d4a711e192..48636be7c7 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -24,6 +24,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/tsdb/encoding" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block" @@ -234,6 +235,7 @@ type BucketStore struct { filterConfig *FilterConfig advLabelSets []storepb.LabelSet enableCompatibilityLabel bool + enableIndexHeader bool } // NewBucketStore creates a new bucket backed store that implements the store API against @@ -252,6 +254,7 @@ func NewBucketStore( blockSyncConcurrency int, filterConfig *FilterConfig, enableCompatibilityLabel bool, + enableIndexHeader bool, ) (*BucketStore, error) { if logger == nil { logger = log.NewNopLogger() @@ -288,6 +291,7 @@ func NewBucketStore( samplesLimiter: NewLimiter(maxSampleCount, metrics.queriesDropped), partitioner: gapBasedPartitioner{maxGapSize: maxGapSize}, enableCompatibilityLabel: enableCompatibilityLabel, + enableIndexHeader: enableIndexHeader, } s.metrics = metrics @@ -447,9 +451,17 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er lset := labels.FromMap(meta.Thanos.Labels) h := lset.Hash() - jr, err := indexheader.NewJSONReader(ctx, s.logger, s.bkt, s.dir, meta.ULID) - if err != nil { - return errors.Wrap(err, "create index header reader") + var indexHeaderReader indexheader.Reader + if s.enableIndexHeader { + indexHeaderReader, err = indexheader.NewBinaryReader(ctx, s.logger, s.bkt, s.dir, meta.ULID) + if err != nil { + return errors.Wrap(err, "create index header reader") + } + } else { + indexHeaderReader, err = indexheader.NewJSONReader(ctx, s.logger, s.bkt, s.dir, meta.ULID) + if err != nil { + return errors.Wrap(err, "create index cache reader") + } } b, err := newBucketBlock( @@ -460,7 +472,7 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er dir, s.indexCache, s.chunkPool, - jr, + indexHeaderReader, s.partitioner, s.metrics.seriesRefetches, ) @@ -1005,7 +1017,10 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR defer runutil.CloseWithLogOnErr(s.logger, indexr, "label values") // Do it via index reader to have pending reader registered correctly. - res := indexr.block.indexHeaderReader.LabelValues(req.Label) + res, err := indexr.block.indexHeaderReader.LabelValues(req.Label) + if err != nil { + return errors.Wrap(err, "index header label values") + } mtx.Lock() sets = append(sets, res) @@ -1301,7 +1316,12 @@ func (r *bucketIndexReader) ExpandedPostings(ms []*labels.Matcher) ([]uint64, er // NOTE: Derived from tsdb.PostingsForMatchers. for _, m := range ms { // Each group is separate to tell later what postings are intersecting with what. - postingGroups = append(postingGroups, toPostingGroup(r.block.indexHeaderReader.LabelValues, m)) + pg, err := toPostingGroup(r.block.indexHeaderReader.LabelValues, m) + if err != nil { + return nil, errors.Wrap(err, "toPostingGroup") + } + + postingGroups = append(postingGroups, pg) } if len(postingGroups) == 0 { @@ -1376,7 +1396,7 @@ func allWithout(p []index.Postings) index.Postings { } // NOTE: Derived from tsdb.postingsForMatcher. index.Merge is equivalent to map duplication. -func toPostingGroup(lvalsFn func(name string) []string, m *labels.Matcher) *postingGroup { +func toPostingGroup(lvalsFn func(name string) ([]string, error), m *labels.Matcher) (*postingGroup, error) { var matchingLabels labels.Labels // If the matcher selects an empty value, it selects all the series which don't @@ -1386,7 +1406,11 @@ func toPostingGroup(lvalsFn func(name string) []string, m *labels.Matcher) *post allName, allValue := index.AllPostingsKey() matchingLabels = append(matchingLabels, labels.Label{Name: allName, Value: allValue}) - for _, val := range lvalsFn(m.Name) { + vals, err := lvalsFn(m.Name) + if err != nil { + return nil, err + } + for _, val := range vals { if !m.Matches(val) { matchingLabels = append(matchingLabels, labels.Label{Name: m.Name, Value: val}) } @@ -1396,24 +1420,29 @@ func toPostingGroup(lvalsFn func(name string) []string, m *labels.Matcher) *post // This is known hack to return all series. // Ask for x != . Allow for that as Prometheus does, // even though it is expensive. - return newPostingGroup(matchingLabels, merge) + return newPostingGroup(matchingLabels, merge), nil } - return newPostingGroup(matchingLabels, allWithout) + return newPostingGroup(matchingLabels, allWithout), nil } // Fast-path for equal matching. if m.Type == labels.MatchEqual { - return newPostingGroup(labels.Labels{{Name: m.Name, Value: m.Value}}, merge) + return newPostingGroup(labels.Labels{{Name: m.Name, Value: m.Value}}, merge), nil + } + + vals, err := lvalsFn(m.Name) + if err != nil { + return nil, err } - for _, val := range lvalsFn(m.Name) { + for _, val := range vals { if m.Matches(val) { matchingLabels = append(matchingLabels, labels.Label{Name: m.Name, Value: val}) } } - return newPostingGroup(matchingLabels, merge) + return newPostingGroup(matchingLabels, merge), nil } type postingPtr struct { @@ -1448,18 +1477,23 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { if err != nil { return errors.Wrap(err, "decode postings") } + g.Fill(j, l) continue } // Cache miss; save pointer for actual posting in index stored in object store. - ptr := r.block.indexHeaderReader.PostingsOffset(key.Name, key.Value) - if ptr == indexheader.NotFoundRange { + ptr, err := r.block.indexHeaderReader.PostingsOffset(key.Name, key.Value) + if err == indexheader.NotFoundRangeErr { // This block does not have any posting for given key. g.Fill(j, index.EmptyPostings()) continue } + if err != nil { + return errors.Wrap(err, "index header PostingsOffset") + } + r.stats.postingsToFetch++ ptrs = append(ptrs, postingPtr{ptr: ptr, groupID: i, keyID: j}) } @@ -1501,21 +1535,21 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { r.mtx.Unlock() for _, p := range ptrs[i:j] { - c := b[p.ptr.Start-start : p.ptr.End-start] - - _, fetchedPostings, err := r.dec.Postings(c) + // index-header can estimate endings, which means we need to resize the endings. + pBytes, err := resizePostings(b[p.ptr.Start-start : p.ptr.End-start]) if err != nil { - return errors.Wrap(err, "read postings list") + return err } r.mtx.Lock() // Return postings and fill LRU cache. - groups[p.groupID].Fill(p.keyID, fetchedPostings) - r.cache.StorePostings(r.ctx, r.block.meta.ULID, groups[p.groupID].keys[p.keyID], c) + // Truncate first 4 bytes which are length of posting. + groups[p.groupID].Fill(p.keyID, newBigEndianPostings(pBytes[4:])) + r.cache.StorePostings(r.ctx, r.block.meta.ULID, groups[p.groupID].keys[p.keyID], pBytes) // If we just fetched it we still have to update the stats for touched postings. r.stats.postingsTouched++ - r.stats.postingsTouchedSizeSum += len(c) + r.stats.postingsTouchedSizeSum += len(pBytes) r.mtx.Unlock() } return nil @@ -1525,6 +1559,70 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { return g.Wait() } +func resizePostings(b []byte) ([]byte, error) { + d := encoding.Decbuf{B: b} + n := d.Be32int() + if d.Err() != nil { + return nil, errors.Wrap(d.Err(), "read postings list") + } + + // 4 for postings number of entries, then 4, foreach each big endian posting. + size := 4 + n*4 + if len(b) < size { + return nil, encoding.ErrInvalidSize + } + return b[:size], nil +} + +// bigEndianPostings implements the Postings interface over a byte stream of +// big endian numbers. +type bigEndianPostings struct { + list []byte + cur uint32 +} + +// TODO(bwplotka): Expose those inside Prometheus. +func newBigEndianPostings(list []byte) *bigEndianPostings { + return &bigEndianPostings{list: list} +} + +func (it *bigEndianPostings) At() uint64 { + return uint64(it.cur) +} + +func (it *bigEndianPostings) Next() bool { + if len(it.list) >= 4 { + it.cur = binary.BigEndian.Uint32(it.list) + it.list = it.list[4:] + return true + } + return false +} + +func (it *bigEndianPostings) Seek(x uint64) bool { + if uint64(it.cur) >= x { + return true + } + + num := len(it.list) / 4 + // Do binary search between current position and end. + i := sort.Search(num, func(i int) bool { + return binary.BigEndian.Uint32(it.list[i*4:]) >= uint32(x) + }) + if i < num { + j := i * 4 + it.cur = binary.BigEndian.Uint32(it.list[j:]) + it.list = it.list[j+4:] + return true + } + it.list = nil + return false +} + +func (it *bigEndianPostings) Err() error { + return nil +} + func (r *bucketIndexReader) PreloadSeries(ids []uint64) error { // Load series from cache, overwriting the list of ids to preload // with the missing ones. diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index ace07aa7b8..531cf73552 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -163,6 +163,7 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m 20, filterConf, true, + true, ) testutil.Ok(t, err) s.store = store @@ -178,7 +179,8 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m return s } -func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { +// TODO(bwplotka): Benchmark Series. +func testBucketStore_e2e(t *testing.T, ctx context.Context, s *storeSuite) { mint, maxt := s.store.TimeRange() testutil.Equals(t, s.minTime, mint) testutil.Equals(t, s.maxTime, maxt) @@ -392,16 +394,18 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { }, }, } { - t.Log("Run ", i) - - srv := newStoreSeriesServer(ctx) - - testutil.Ok(t, s.store.Series(tcase.req, srv)) - testutil.Equals(t, len(tcase.expected), len(srv.SeriesSet)) - - for i, s := range srv.SeriesSet { - testutil.Equals(t, tcase.expected[i], s.Labels) - testutil.Equals(t, tcase.expectedChunkLen, len(s.Chunks)) + if ok := t.Run(fmt.Sprint(i), func(t *testing.T) { + srv := newStoreSeriesServer(ctx) + + testutil.Ok(t, s.store.Series(tcase.req, srv)) + testutil.Equals(t, len(tcase.expected), len(srv.SeriesSet)) + + for i, s := range srv.SeriesSet { + testutil.Equals(t, tcase.expected[i], s.Labels) + testutil.Equals(t, tcase.expectedChunkLen, len(s.Chunks)) + } + }); !ok { + return } } } @@ -417,27 +421,36 @@ func TestBucketStore_e2e(t *testing.T) { s := prepareStoreWithTestBlocks(t, dir, bkt, false, 0, emptyRelabelConfig, allowAllFilterConf) - t.Log("Test with no index cache") - s.cache.SwapWith(noopCache{}) - testBucketStore_e2e(t, ctx, s) + if ok := t.Run("no index cache", func(t *testing.T) { + s.cache.SwapWith(noopCache{}) + testBucketStore_e2e(t, ctx, s) + }); !ok { + return + } - t.Log("Test with large, sufficient index cache") - indexCache, err := storecache.NewInMemoryIndexCacheWithConfig(s.logger, nil, storecache.InMemoryIndexCacheConfig{ - MaxItemSize: 1e5, - MaxSize: 2e5, - }) - testutil.Ok(t, err) - s.cache.SwapWith(indexCache) - testBucketStore_e2e(t, ctx, s) + if ok := t.Run("with large, sufficient index cache", func(t *testing.T) { + indexCache, err := storecache.NewInMemoryIndexCacheWithConfig(s.logger, nil, storecache.InMemoryIndexCacheConfig{ + MaxItemSize: 1e5, + MaxSize: 2e5, + }) + testutil.Ok(t, err) + s.cache.SwapWith(indexCache) + testBucketStore_e2e(t, ctx, s) + }); !ok { + return + } - t.Log("Test with small index cache") - indexCache2, err := storecache.NewInMemoryIndexCacheWithConfig(s.logger, nil, storecache.InMemoryIndexCacheConfig{ - MaxItemSize: 50, - MaxSize: 100, - }) - testutil.Ok(t, err) - s.cache.SwapWith(indexCache2) - testBucketStore_e2e(t, ctx, s) + if ok := t.Run("with small index cache", func(t *testing.T) { + indexCache2, err := storecache.NewInMemoryIndexCacheWithConfig(s.logger, nil, storecache.InMemoryIndexCacheConfig{ + MaxItemSize: 50, + MaxSize: 100, + }) + testutil.Ok(t, err) + s.cache.SwapWith(indexCache2) + testBucketStore_e2e(t, ctx, s) + }); !ok { + return + } }) } @@ -520,7 +533,6 @@ func TestBucketStore_TimePartitioning_e2e(t *testing.T) { testutil.Equals(t, len(expectedLabels), len(srv.SeriesSet)) for i, s := range srv.SeriesSet { - fmt.Println(s.Labels) testutil.Equals(t, expectedLabels[i], s.Labels) // prepareTestBlocks makes 3 chunks containing 2 hour data, diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index cb861347a2..dbef61e29a 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -3,7 +3,6 @@ package store import ( "bytes" "context" - "fmt" "io" "io/ioutil" "math" @@ -451,6 +450,7 @@ func TestBucketStore_Info(t *testing.T) { 20, allowAllFilterConf, true, + true, ) testutil.Ok(t, err) @@ -468,17 +468,26 @@ type recorder struct { mtx sync.Mutex objstore.Bucket - touched []string + getRangeTouched []string + getTouched []string } func (r *recorder) Get(ctx context.Context, name string) (io.ReadCloser, error) { r.mtx.Lock() defer r.mtx.Unlock() - r.touched = append(r.touched, name) + r.getTouched = append(r.getTouched, name) return r.Bucket.Get(ctx, name) } +func (r *recorder) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { + r.mtx.Lock() + defer r.mtx.Unlock() + + r.getRangeTouched = append(r.getRangeTouched, name) + return r.Bucket.GetRange(ctx, name, off, length) +} + func TestBucketStore_Sharding(t *testing.T) { ctx := context.Background() logger := log.NewNopLogger() @@ -691,7 +700,9 @@ func testSharding(t *testing.T, reuseDisk string, bkt objstore.Bucket, all ...ul false, 20, allowAllFilterConf, - true) + true, + true, + ) testutil.Ok(t, err) testutil.Ok(t, bucketStore.InitialSync(context.Background())) @@ -718,16 +729,17 @@ func testSharding(t *testing.T, reuseDisk string, bkt objstore.Bucket, all ...ul // Regression test: https://github.com/thanos-io/thanos/issues/1664 // Sort records. We load blocks concurrently so operations might be not ordered. - sort.Strings(rec.touched) + sort.Strings(rec.getRangeTouched) - fmt.Println(cached, sc.expectedIDs, all, rec.touched) + // With binary header nothing should be downloaded fully. + testutil.Equals(t, []string(nil), rec.getTouched) if reuseDisk != "" { - testutil.Equals(t, expectedTouchedBlockOps(all, sc.expectedIDs, cached), rec.touched) + testutil.Equals(t, expectedTouchedBlockOps(all, sc.expectedIDs, cached), rec.getRangeTouched) cached = sc.expectedIDs return } - testutil.Equals(t, expectedTouchedBlockOps(all, sc.expectedIDs, nil), rec.touched) + testutil.Equals(t, expectedTouchedBlockOps(all, sc.expectedIDs, nil), rec.getRangeTouched) }) } } @@ -756,8 +768,11 @@ func expectedTouchedBlockOps(all []ulid.ULID, expected []ulid.ULID, cached []uli if found { ops = append(ops, - path.Join(id.String(), block.IndexCacheFilename), - path.Join(id.String(), block.IndexFilename), + // To create binary header we touch part of index few times. + path.Join(id.String(), block.IndexFilename), // Version. + path.Join(id.String(), block.IndexFilename), // TOC. + path.Join(id.String(), block.IndexFilename), // Symbols. + path.Join(id.String(), block.IndexFilename), // PostingOffsets. ) } } diff --git a/pkg/testutil/copy.go b/pkg/testutil/copy.go new file mode 100644 index 0000000000..c2c8bd480d --- /dev/null +++ b/pkg/testutil/copy.go @@ -0,0 +1,49 @@ +package testutil + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/pkg/errors" +) + +func Copy(t testing.TB, src, dst string) { + Ok(t, copyRecursive(src, dst)) +} + +func copyRecursive(src, dst string) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + + if info.IsDir() { + return os.MkdirAll(filepath.Join(dst, relPath), os.ModePerm) + } + + if !info.Mode().IsRegular() { + return errors.Errorf("%s is not a regular file", path) + } + + source, err := os.Open(path) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(filepath.Join(dst, relPath)) + if err != nil { + return err + } + defer destination.Close() + _, err = io.Copy(destination, source) + return err + }) +} diff --git a/test/e2e/spinup_test.go b/test/e2e/spinup_test.go index d51a51a60a..06b3bd3dbf 100644 --- a/test/e2e/spinup_test.go +++ b/test/e2e/spinup_test.go @@ -269,6 +269,7 @@ func storeGateway(http, grpc address, bucketConfig []byte, relabelConfig []byte) // Accelerated sync time for quicker test (3m by default). "--sync-block-duration", "5s", "--selector.relabel-config", string(relabelConfig), + "--experimental.enable-index-header", )), nil }, } From 9fd5974da0bd4036d43532e845def58ce0030d68 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Tue, 21 Jan 2020 14:53:07 +0100 Subject: [PATCH 182/257] mixin/thanos: fix typo in alert name (#2017) Signed-off-by: Simon Pasquier --- examples/alerts/alerts.md | 4 ++-- examples/alerts/alerts.yaml | 4 ++-- mixin/thanos/alerts/ruler.libsonnet | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index 41e135b24c..d603e751b2 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -85,7 +85,7 @@ rules: for: 5m labels: severity: critical -- alert: ThanosRulerHighRuleExaluationFailures +- alert: ThanosRulerHighRuleEvaluationFailures annotations: message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to evaluate rules. expr: | @@ -98,7 +98,7 @@ rules: for: 5m labels: severity: warning -- alert: ThanosRulerHighRuleExaluationWarnings +- alert: ThanosRulerHighRuleEvaluationWarnings annotations: message: Thanos Ruler {{$labels.job}} {{$labels.pod}} has high number of evaluation warnings. diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index 0eb4fda1d1..434d741d58 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -296,7 +296,7 @@ groups: for: 5m labels: severity: critical - - alert: ThanosRulerHighRuleExaluationFailures + - alert: ThanosRulerHighRuleEvaluationFailures annotations: message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to evaluate rules. @@ -310,7 +310,7 @@ groups: for: 5m labels: severity: warning - - alert: ThanosRulerHighRuleExaluationWarnings + - alert: ThanosRulerHighRuleEvaluationWarnings annotations: message: Thanos Ruler {{$labels.job}} {{$labels.pod}} has high number of evaluation warnings. diff --git a/mixin/thanos/alerts/ruler.libsonnet b/mixin/thanos/alerts/ruler.libsonnet index 08de10a23b..16d5c4e486 100644 --- a/mixin/thanos/alerts/ruler.libsonnet +++ b/mixin/thanos/alerts/ruler.libsonnet @@ -36,7 +36,7 @@ }, }, { - alert: 'ThanosRulerHighRuleExaluationFailures', + alert: 'ThanosRulerHighRuleEvaluationFailures', annotations: { message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to evaluate rules.', }, @@ -55,7 +55,7 @@ }, }, { - alert: 'ThanosRulerHighRuleExaluationWarnings', + alert: 'ThanosRulerHighRuleEvaluationWarnings', annotations: { message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} has high number of evaluation warnings.', }, From b1abfebfa06382363dada50a0883b0b25d2cfeca Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Wed, 22 Jan 2020 04:14:22 +1300 Subject: [PATCH 183/257] Clear up wording around 0d retention times (#2006) Signed-off-by: Ben Moskovitz --- cmd/thanos/compact.go | 6 +++--- docs/components/compact.md | 13 +++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 4ff22f0020..5832625158 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -96,9 +96,9 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) { consistencyDelay := modelDuration(cmd.Flag("consistency-delay", fmt.Sprintf("Minimum age of fresh (non-compacted) blocks before they are being processed. Malformed blocks older than the maximum of consistency-delay and %v will be removed.", compact.PartialUploadThresholdAge)). Default("30m")) - retentionRaw := modelDuration(cmd.Flag("retention.resolution-raw", "How long to retain raw samples in bucket. 0d - disables this retention").Default("0d")) - retention5m := modelDuration(cmd.Flag("retention.resolution-5m", "How long to retain samples of resolution 1 (5 minutes) in bucket. 0d - disables this retention").Default("0d")) - retention1h := modelDuration(cmd.Flag("retention.resolution-1h", "How long to retain samples of resolution 2 (1 hour) in bucket. 0d - disables this retention").Default("0d")) + retentionRaw := modelDuration(cmd.Flag("retention.resolution-raw", "How long to retain raw samples in bucket. Setting this to 0d will retain samples of this resolution forever").Default("0d")) + retention5m := modelDuration(cmd.Flag("retention.resolution-5m", "How long to retain samples of resolution 1 (5 minutes) in bucket. Setting this to 0d will retain samples of this resolution forever").Default("0d")) + retention1h := modelDuration(cmd.Flag("retention.resolution-1h", "How long to retain samples of resolution 2 (1 hour) in bucket. Setting this to 0d will retain samples of this resolution forever").Default("0d")) wait := cmd.Flag("wait", "Do not exit after all compactions have been processed and wait for new work."). Short('w').Bool() diff --git a/docs/components/compact.md b/docs/components/compact.md index 440ad9f1c9..d3c3b78a55 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -47,6 +47,8 @@ There's also a case when you might want to disable downsampling at all with `deb Ideally, you will have equal retention set (or no retention at all) to all resolutions which allow both "zoom in" capabilities as well as performant long ranges queries. Since object storages are usually quite cheap, storage size might not matter that much, unless your goal with thanos is somewhat very specific and you know exactly what you're doing. +Not setting this flag, or setting it to `0d`, i.e. `--retention.resolution-X=0d`, will mean that samples at the `X` resolution level will be kept forever. + ## Storage space consumption In fact, downsampling doesn't save you any space but instead it adds 2 more blocks for each raw block which are only slightly smaller or relatively similar size to raw block. This is required by internal downsampling implementation which to be mathematically correct holds various aggregations. This means that downsampling can increase the size of your storage a bit (~3x), but it gives massive advantage on querying long ranges. @@ -105,14 +107,17 @@ Flags: older than the maximum of consistency-delay and 48h0m0s will be removed. --retention.resolution-raw=0d - How long to retain raw samples in bucket. 0d - - disables this retention + How long to retain raw samples in bucket. Setting + this to 0d will retain samples of this resolution + forever --retention.resolution-5m=0d How long to retain samples of resolution 1 (5 - minutes) in bucket. 0d - disables this retention + minutes) in bucket. Setting this to 0d will + retain samples of this resolution forever --retention.resolution-1h=0d How long to retain samples of resolution 2 (1 - hour) in bucket. 0d - disables this retention + hour) in bucket. Setting this to 0d will retain + samples of this resolution forever -w, --wait Do not exit after all compactions have been processed and wait for new work. --downsampling.disable Disables downsampling. This is not recommended as From dbfe6f076007991b6e54ce0570c6a62d51cae448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Tue, 21 Jan 2020 20:33:16 +0200 Subject: [PATCH 184/257] Fix revert metric name regression (#2005) (#2019) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kemal Akkoyun Signed-off-by: Giedrius Statkevičius Co-authored-by: Kemal Akkoyun --- examples/alerts/alerts.md | 4 ++-- examples/alerts/alerts.yaml | 4 ++-- mixin/thanos/alerts/querier.libsonnet | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index d603e751b2..a3b7945d21 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -308,9 +308,9 @@ rules: message: Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries. expr: | ( - sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) / - sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) > 1 ) for: 15m diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index 434d741d58..eb69ba9a1d 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -112,9 +112,9 @@ groups: message: Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries. expr: | ( - sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) / - sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) > 1 ) for: 15m diff --git a/mixin/thanos/alerts/querier.libsonnet b/mixin/thanos/alerts/querier.libsonnet index 38705aacf3..f50100161b 100644 --- a/mixin/thanos/alerts/querier.libsonnet +++ b/mixin/thanos/alerts/querier.libsonnet @@ -86,9 +86,9 @@ }, expr: ||| ( - sum by (job) (rate(thanos_query_store_apis_dns_failures_total{%(selector)s}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{%(selector)s}[5m])) / - sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{%(selector)s}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{%(selector)s}[5m])) > 1 ) ||| % thanos.querier, From b1302218abfdbeff879581a2a9a18c16ff202f0e Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 23 Jan 2020 03:50:33 +0800 Subject: [PATCH 185/257] Add env comment (#2023) * Add env comment Signed-off-by: Xiang Dai <764524258@qq.com> * feedback Signed-off-by: Xiang Dai <764524258@qq.com> --- docs/components/sidecar.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 4823f7cbe0..6fd960343f 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -43,7 +43,7 @@ Thanos can watch changes in Prometheus configuration and refresh Prometheus conf You can configure watching for changes in directory via `--reloader.rule-dir=DIR_NAME` flag. -Thanos sidecar can watch `--reloader.config-file=CONFIG_FILE` configuration file, evaluate environment variables found in there, and produce generated config in `--reloader.config-envsubst-file=OUT_CONFIG_FILE` file. +Thanos sidecar can watch `--reloader.config-file=CONFIG_FILE` configuration file, replace environment variables found in there in `$(VARIABLE)` format, and produce generated config in `--reloader.config-envsubst-file=OUT_CONFIG_FILE` file. ## Example basic deployment From 43f659a8af48edfecc8b2de776744d95a6d6707b Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 23 Jan 2020 02:36:34 -0500 Subject: [PATCH 186/257] Fix invalid start/end params in series api (#2015) * fix invalid start/end params in series api Signed-off-by: yeya24 * add testcase for serieslabels time range Signed-off-by: yeya24 --- pkg/model/timeduration.go | 2 +- pkg/store/prometheus.go | 11 +++++++++-- pkg/store/prometheus_test.go | 38 +++++++++++++++++++++++++++++------- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/pkg/model/timeduration.go b/pkg/model/timeduration.go index bbe766043f..a40a6a78f4 100644 --- a/pkg/model/timeduration.go +++ b/pkg/model/timeduration.go @@ -42,7 +42,7 @@ func (tdv *TimeOrDurationValue) Set(s string) error { return nil } -// String returns either tume or duration. +// String returns either time or duration. func (tdv *TimeOrDurationValue) String() string { switch { case tdv.Time != nil: diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 41fa52e3e6..976b33f47f 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -12,8 +12,10 @@ import ( "net/url" "path" "sort" + "strconv" "strings" "sync" + "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -23,6 +25,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/version" "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage/remote" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -663,9 +666,9 @@ func (p *PrometheusStore) seriesLabels(ctx context.Context, matchers []storepb.L } q.Add("match[]", metric) + q.Add("start", formatTime(timestamp.Time(startTime))) + q.Add("end", formatTime(timestamp.Time(endTime))) u.RawQuery = q.Encode() - q.Add("start", string(startTime)) - q.Add("end", string(endTime)) req, err := http.NewRequest("GET", u.String(), nil) if err != nil { @@ -715,3 +718,7 @@ func (p *PrometheusStore) seriesLabels(ctx context.Context, matchers []storepb.L return m.Data, nil } + +func formatTime(t time.Time) string { + return strconv.FormatFloat(float64(t.Unix())+float64(t.Nanosecond())/1e9, 'f', -1, 64) +} diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index bf6fee5cc7..a3adf05218 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -199,6 +199,8 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { testutil.Ok(t, err) _, err = a.Add(labels.FromStrings("a", "d", "job", "test"), baseT+300, 3) testutil.Ok(t, err) + _, err = a.Add(labels.FromStrings("job", "test"), baseT+400, 4) + testutil.Ok(t, err) testutil.Ok(t, a.Commit()) ctx, cancel := context.WithCancel(context.Background()) @@ -209,14 +211,13 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - limitMinT := int64(0) - proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, + promStore, err := NewPrometheusStore(nil, nil, u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return limitMinT, -1 }) // Maxt does not matter. + func() (int64, int64) { return math.MinInt64/1000 + 62135596801, math.MaxInt64/1000 - 62135596801 }) testutil.Ok(t, err) { - res, err := proxy.seriesLabels(ctx, []storepb.LabelMatcher{ + res, err := promStore.seriesLabels(ctx, []storepb.LabelMatcher{ {Type: storepb.LabelMatcher_EQ, Name: "a", Value: "b"}, }, baseT, baseT+300) testutil.Ok(t, err) @@ -225,7 +226,7 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { testutil.Equals(t, labels.FromMap(res[0]), labels.Labels{{Name: "a", Value: "b"}}) } { - res, err := proxy.seriesLabels(ctx, []storepb.LabelMatcher{ + res, err := promStore.seriesLabels(ctx, []storepb.LabelMatcher{ {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "foo"}, }, baseT, baseT+300) testutil.Ok(t, err) @@ -233,7 +234,7 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { testutil.Equals(t, len(res), 0) } { - res, err := proxy.seriesLabels(ctx, []storepb.LabelMatcher{ + res, err := promStore.seriesLabels(ctx, []storepb.LabelMatcher{ {Type: storepb.LabelMatcher_NEQ, Name: "a", Value: "b"}, {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "test"}, }, baseT, baseT+300) @@ -247,7 +248,7 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { } } { - res, err := proxy.seriesLabels(ctx, []storepb.LabelMatcher{ + res, err := promStore.seriesLabels(ctx, []storepb.LabelMatcher{ {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "test"}, }, baseT, baseT+300) testutil.Ok(t, err) @@ -259,6 +260,29 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { testutil.Equals(t, labels.FromMap(r).Get("job"), "test") } } + { + res, err := promStore.seriesLabels(ctx, []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "test"}, + }, baseT+400, baseT+400) + testutil.Ok(t, err) + + // In baseT + 400 we can just get one series. + testutil.Equals(t, len(res), 1) + testutil.Equals(t, len(res[0]), 1) + testutil.Equals(t, labels.FromMap(res[0]).Get("job"), "test") + } + // This test case is to test when start time and end time is not specified. + { + minTime, maxTime := promStore.timestamps() + res, err := promStore.seriesLabels(ctx, []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "test"}, + }, minTime, maxTime) + testutil.Ok(t, err) + testutil.Equals(t, len(res), 3) + for _, r := range res { + testutil.Equals(t, labels.FromMap(r).Get("job"), "test") + } + } } func TestPrometheusStore_LabelValues_e2e(t *testing.T) { From bc8b925c728b6046b41e10d1d27f41e58a3d0aee Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 23 Jan 2020 09:26:19 +0100 Subject: [PATCH 187/257] pkg/alert: fix configuration from "legacy" flags (#2026) Signed-off-by: Simon Pasquier --- pkg/alert/config.go | 3 ++- pkg/alert/config_test.go | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/alert/config.go b/pkg/alert/config.go index e8398b4abb..6e402091d4 100644 --- a/pkg/alert/config.go +++ b/pkg/alert/config.go @@ -123,6 +123,7 @@ func BuildAlertmanagerConfig(address string, timeout time.Duration) (Alertmanage Scheme: scheme, StaticAddresses: []string{host}, }, - Timeout: model.Duration(timeout), + Timeout: model.Duration(timeout), + APIVersion: APIv1, }, nil } diff --git a/pkg/alert/config_test.go b/pkg/alert/config_test.go index 9dd7ddab19..ea7c3de773 100644 --- a/pkg/alert/config_test.go +++ b/pkg/alert/config_test.go @@ -55,6 +55,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { StaticAddresses: []string{"localhost:9093"}, Scheme: "http", }, + APIVersion: APIv1, }, }, { @@ -64,6 +65,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { StaticAddresses: []string{"am.example.com"}, Scheme: "https", }, + APIVersion: APIv1, }, }, { @@ -73,6 +75,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { StaticAddresses: []string{"dns+localhost:9093"}, Scheme: "http", }, + APIVersion: APIv1, }, }, { @@ -82,6 +85,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { StaticAddresses: []string{"dnssrv+localhost"}, Scheme: "http", }, + APIVersion: APIv1, }, }, { @@ -91,6 +95,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { StaticAddresses: []string{"localhost"}, Scheme: "ssh+http", }, + APIVersion: APIv1, }, }, { @@ -101,6 +106,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { Scheme: "https", PathPrefix: "/path/prefix/", }, + APIVersion: APIv1, }, }, { @@ -116,6 +122,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { StaticAddresses: []string{"localhost:9093"}, Scheme: "http", }, + APIVersion: APIv1, }, }, { From 860bbaf4fbc9845bc093ce6a39b7316bf20d1e50 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Thu, 23 Jan 2020 13:21:17 +0100 Subject: [PATCH 188/257] pkg/store: Expose metric about empty stream responses in proxy store (#2030) Signed-off-by: Frederic Branczyk --- CHANGELOG.md | 1 + cmd/thanos/query.go | 2 +- pkg/store/proxy.go | 36 +++++++++++++++++++++++++++++++++++- pkg/store/proxy_test.go | 7 +++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8feac8514..fe71361ade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`. - [#1939](https://github.com/thanos-io/thanos/pull/1939) Ruler: Add TLS and authentication support for query endpoints with the `--query.config` and `--query.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. - [#1982](https://github.com/thanos-io/thanos/pull/1982) Ruler: Add support for Alertmanager v2 API endpoints. +- #2030 Query: Add `thanos_proxy_store_empty_stream_responses_total` metric for number of empty responses from stores. ### Changed diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 349e9bf2ea..b2edceceef 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -235,7 +235,7 @@ func runQuery( dialOpts, unhealthyStoreTimeout, ) - proxy = store.NewProxyStore(logger, stores.Get, component.Query, selectorLset, storeResponseTimeout) + proxy = store.NewProxyStore(logger, reg, stores.Get, component.Query, selectorLset, storeResponseTimeout) queryableCreator = query.NewQueryableCreator(logger, proxy) engine = promql.NewEngine( promql.EngineOpts{ diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 574c5e0e4b..b38f2c952b 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -15,6 +15,7 @@ import ( grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/store/storepb" @@ -49,12 +50,34 @@ type ProxyStore struct { selectorLabels labels.Labels responseTimeout time.Duration + metrics *proxyStoreMetrics +} + +type proxyStoreMetrics struct { + emptyStreamResponses prometheus.Counter +} + +func newProxyStoreMetrics(reg prometheus.Registerer) *proxyStoreMetrics { + var m proxyStoreMetrics + + m.emptyStreamResponses = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "thanos_proxy_store_empty_stream_responses_total", + Help: "Total number of empty responses received.", + }) + + if reg != nil { + reg.MustRegister( + m.emptyStreamResponses, + ) + } + return &m } // NewProxyStore returns a new ProxyStore that uses the given clients that implements storeAPI to fan-in all series to the client. // Note that there is no deduplication support. Deduplication should be done on the highest level (just before PromQL). func NewProxyStore( logger log.Logger, + reg prometheus.Registerer, stores func() []Client, component component.StoreAPI, selectorLabels labels.Labels, @@ -64,12 +87,14 @@ func NewProxyStore( logger = log.NewNopLogger() } + metrics := newProxyStoreMetrics(reg) s := &ProxyStore{ logger: logger, stores: stores, component: component, selectorLabels: selectorLabels, responseTimeout: responseTimeout, + metrics: metrics, } return s } @@ -260,7 +285,7 @@ func (s *ProxyStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSe // Schedule streamSeriesSet that translates gRPC streamed response // into seriesSet (if series) or respCh if warnings. seriesSet = append(seriesSet, startStreamSeriesSet(seriesCtx, s.logger, closeSeries, - wg, sc, respSender, st.String(), !r.PartialResponseDisabled, s.responseTimeout)) + wg, sc, respSender, st.String(), !r.PartialResponseDisabled, s.responseTimeout, s.metrics.emptyStreamResponses)) } level.Debug(s.logger).Log("msg", strings.Join(storeDebugMsgs, ";")) @@ -330,6 +355,7 @@ func startStreamSeriesSet( name string, partialResponse bool, responseTimeout time.Duration, + emptyStreamResponses prometheus.Counter, ) *streamSeriesSet { s := &streamSeriesSet{ ctx: ctx, @@ -348,6 +374,12 @@ func startStreamSeriesSet( defer wg.Done() defer close(s.recvCh) + numResponses := 0 + defer func() { + if numResponses == 0 { + emptyStreamResponses.Inc() + } + }() for { r, err := s.stream.Recv() @@ -368,6 +400,8 @@ func startStreamSeriesSet( return } + numResponses++ + if w := r.GetWarning(); w != "" { s.warnCh.send(storepb.NewWarnSeriesResponse(errors.New(w))) continue diff --git a/pkg/store/proxy_test.go b/pkg/store/proxy_test.go index d19bf58f9a..ce26e2ba41 100644 --- a/pkg/store/proxy_test.go +++ b/pkg/store/proxy_test.go @@ -53,6 +53,7 @@ func TestProxyStore_Info(t *testing.T) { defer cancel() q := NewProxyStore(nil, + nil, func() []Client { return nil }, component.Query, nil, 0*time.Second, @@ -438,6 +439,7 @@ func TestProxyStore_Series(t *testing.T) { if ok := t.Run(tc.title, func(t *testing.T) { q := NewProxyStore(nil, + nil, func() []Client { return tc.storeAPIs }, component.Query, tc.selectorLabels, @@ -560,6 +562,7 @@ func TestProxyStore_SeriesSlowStores(t *testing.T) { } { if ok := t.Run(tc.title, func(t *testing.T) { q := NewProxyStore(nil, + nil, func() []Client { return tc.storeAPIs }, component.Query, tc.selectorLabels, @@ -602,6 +605,7 @@ func TestProxyStore_Series_RequestParamsProxied(t *testing.T) { }, } q := NewProxyStore(nil, + nil, func() []Client { return cls }, component.Query, nil, @@ -661,6 +665,7 @@ func TestProxyStore_Series_RegressionFillResponseChannel(t *testing.T) { } q := NewProxyStore(nil, + nil, func() []Client { return cls }, component.Query, labels.FromStrings("fed", "a"), @@ -699,6 +704,7 @@ func TestProxyStore_LabelValues(t *testing.T) { }}, } q := NewProxyStore(nil, + nil, func() []Client { return cls }, component.Query, nil, @@ -801,6 +807,7 @@ func TestProxyStore_LabelNames(t *testing.T) { } { if ok := t.Run(tc.title, func(t *testing.T) { q := NewProxyStore( + nil, nil, func() []Client { return tc.storeAPIs }, component.Query, From bb488307f5ac979544e89a594f24791846f1d49e Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 23 Jan 2020 09:51:37 -0500 Subject: [PATCH 189/257] add _total suffix to counter metrics (#2021) Signed-off-by: yeya24 --- CHANGELOG.md | 1 + cmd/thanos/query.go | 2 +- cmd/thanos/rule.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe71361ade..c83a0646c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Changed - [#1933](https://github.com/thanos-io/thanos/pull/1933) Add a flag `--tsdb.wal-compression` to configure whether to enable tsdb wal compression in ruler and receiver. +- [#2021](https://github.com/thanos-io/thanos/pull/2021) Rename metric `thanos_query_duplicated_store_address` to `thanos_query_duplicated_store_addresses_total` and `thanos_rule_duplicated_query_address` to `thanos_rule_duplicated_query_addresses_total`. ## [v0.10.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0) - 2020.01.13 diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index b2edceceef..9d3412db31 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -201,7 +201,7 @@ func runQuery( ) error { // TODO(bplotka in PR #513 review): Move arguments into struct. duplicatedStores := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "thanos_query_duplicated_store_address", + Name: "thanos_query_duplicated_store_addresses_total", Help: "The number of times a duplicated store addresses is detected from the different configs in query", }) reg.MustRegister(duplicatedStores) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index d734da03d3..1d38974b9e 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -246,7 +246,7 @@ func runRule( Help: "Timestamp of the last successful configuration reload.", }) duplicatedQuery := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "thanos_rule_duplicated_query_address", + Name: "thanos_rule_duplicated_query_addresses_total", Help: "The number of times a duplicated query addresses is detected from the different configs in rule", }) rulesLoaded := prometheus.NewGaugeVec( From de82fec786e3bd2c20f99a3674f5bed6a33028c4 Mon Sep 17 00:00:00 2001 From: Sylvain Rabot Date: Thu, 23 Jan 2020 17:45:20 +0100 Subject: [PATCH 190/257] Use golang 1.13.6 (#2025) Signed-off-by: Sylvain Rabot --- .circleci/config.yml | 2 +- .promu.yml | 2 +- Dockerfile.multi-stage | 2 +- Dockerfile.thanos-ci | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a2387bcc71..f18573d037 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ jobs: test: docker: # Build by Thanos make docker-ci - - image: quay.io/thanos/thanos-ci:v0.2.0 + - image: quay.io/thanos/thanos-ci:v0.3.0 working_directory: /go/src/github.com/thanos-io/thanos environment: GO111MODULE: 'on' diff --git a/.promu.yml b/.promu.yml index 2b2188d562..7139c31d39 100644 --- a/.promu.yml +++ b/.promu.yml @@ -1,5 +1,5 @@ go: - version: 1.13.1 + version: 1.13.6 repository: path: github.com/thanos-io/thanos build: diff --git a/Dockerfile.multi-stage b/Dockerfile.multi-stage index a86c5e67fb..dbcc5aa3f0 100644 --- a/Dockerfile.multi-stage +++ b/Dockerfile.multi-stage @@ -1,4 +1,4 @@ -FROM golang:1.13.1-alpine3.10 as builder +FROM golang:1.13.6-alpine3.11 as builder ADD . $GOPATH/src/github.com/thanos-io/thanos WORKDIR $GOPATH/src/github.com/thanos-io/thanos diff --git a/Dockerfile.thanos-ci b/Dockerfile.thanos-ci index cf9e39a077..4c12365193 100644 --- a/Dockerfile.thanos-ci +++ b/Dockerfile.thanos-ci @@ -1,5 +1,5 @@ # Available from https://hub.docker.com/r/circleci/golang/ -FROM circleci/golang:1.13.1 +FROM circleci/golang:1.13.6 ENV GOBIN=/go/bin From 560816bd531b107bb8e46c4010543970aaf8236d Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Fri, 24 Jan 2020 02:15:44 -0500 Subject: [PATCH 191/257] use gauge.SetToCurrentTime (#2036) Signed-off-by: yeya24 --- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 4 ++-- pkg/objstore/objstore.go | 3 +-- pkg/receive/config.go | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 1d38974b9e..5fcd9c2faa 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -505,7 +505,7 @@ func runRule( } configSuccess.Set(1) - configSuccessTime.Set(float64(time.Now().UnixNano()) / 1e9) + configSuccessTime.SetToCurrentTime() rulesLoaded.Reset() for _, group := range ruleMgr.RuleGroups() { diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 52161f32c0..bd662e87e3 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -213,7 +213,7 @@ func runSidecar( ) promUp.Set(1) statusProber.Ready() - lastHeartbeat.Set(float64(time.Now().UnixNano()) / 1e9) + lastHeartbeat.SetToCurrentTime() return nil }) if err != nil { @@ -235,7 +235,7 @@ func runSidecar( promUp.Set(0) } else { promUp.Set(1) - lastHeartbeat.Set(float64(time.Now().UnixNano()) / 1e9) + lastHeartbeat.SetToCurrentTime() } return nil diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index 5a70d885dc..ba6e775f82 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -316,8 +316,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err if err != nil { b.opsFailures.WithLabelValues(op).Inc() } else { - // TODO: Use SetToCurrentTime() once we update the Prometheus client_golang. - b.lastSuccessfullUploadTime.WithLabelValues(b.bkt.Name()).Set(float64(time.Now().UnixNano()) / 1e9) + b.lastSuccessfullUploadTime.WithLabelValues(b.bkt.Name()).SetToCurrentTime() } b.ops.WithLabelValues(op).Inc() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) diff --git a/pkg/receive/config.go b/pkg/receive/config.go index f576be5fa9..24be94b856 100644 --- a/pkg/receive/config.go +++ b/pkg/receive/config.go @@ -224,7 +224,7 @@ func (cw *ConfigWatcher) refresh(ctx context.Context) { // Save the last known configuration. cw.last = config cw.successGauge.Set(1) - cw.lastSuccessTimeGauge.Set(float64(time.Now().Unix())) + cw.lastSuccessTimeGauge.SetToCurrentTime() cw.hashGauge.Set(hashAsMetricValue(cfgContent)) for _, c := range config { From 49b67e7d2907a3457061e96d7742eadac5288bfa Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 24 Jan 2020 10:08:34 +0100 Subject: [PATCH 192/257] Cut 0.10.1 (#2035) (#2037) Signed-off-by: Bartlomiej Plotka --- CHANGELOG.md | 7 +++++++ VERSION | 2 +- examples/alerts/alerts.md | 4 ++-- examples/alerts/alerts.yaml | 4 ++-- mixin/thanos/alerts/querier.libsonnet | 4 ++-- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c83a0646c4..09776497e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,13 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1933](https://github.com/thanos-io/thanos/pull/1933) Add a flag `--tsdb.wal-compression` to configure whether to enable tsdb wal compression in ruler and receiver. - [#2021](https://github.com/thanos-io/thanos/pull/2021) Rename metric `thanos_query_duplicated_store_address` to `thanos_query_duplicated_store_addresses_total` and `thanos_rule_duplicated_query_address` to `thanos_rule_duplicated_query_addresses_total`. +## [v0.10.1](https://github.com/thanos-io/thanos/releases/tag/v0.10.1) - 2020.01.24 + +### Fixed + +- [#2015](https://github.com/thanos-io/thanos/pull/2015) Sidecar: Querier /api/v1/series bug fixed when time range was ignored inside sidecar. +The bug was noticeable for example when using Grafana template variables. + ## [v0.10.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0) - 2020.01.13 ### Fixed diff --git a/VERSION b/VERSION index 29b2d3ea50..539f9fc668 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.10.0-dev +0.10.1-dev diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index a3b7945d21..d603e751b2 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -308,9 +308,9 @@ rules: message: Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries. expr: | ( - sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) / - sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) > 1 ) for: 15m diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index eb69ba9a1d..434d741d58 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -112,9 +112,9 @@ groups: message: Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries. expr: | ( - sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) / - sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) > 1 ) for: 15m diff --git a/mixin/thanos/alerts/querier.libsonnet b/mixin/thanos/alerts/querier.libsonnet index f50100161b..38705aacf3 100644 --- a/mixin/thanos/alerts/querier.libsonnet +++ b/mixin/thanos/alerts/querier.libsonnet @@ -86,9 +86,9 @@ }, expr: ||| ( - sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{%(selector)s}[5m])) + sum by (job) (rate(thanos_query_store_apis_dns_failures_total{%(selector)s}[5m])) / - sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{%(selector)s}[5m])) + sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{%(selector)s}[5m])) > 1 ) ||| % thanos.querier, From ee1ca84d2e0f121d8fb5dca586630b4ef275ea3c Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 24 Jan 2020 12:27:47 +0100 Subject: [PATCH 193/257] Added Ben Ye to Triage. (#2038) Signed-off-by: Bartlomiej Plotka --- MAINTAINERS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index fba1323272..08ad64261d 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -8,7 +8,7 @@ | Giedrius Statkevičius | giedriuswork@gmail.com | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS) | AdForm | | Povilas Versockas | p.versockas@gmail.com | `@povilasv` | [@povilasv](https://github.com/povilasv) | Utility Warehouse | | Matthias Loibl | mail@matthiasloibl.com | `@metalmatze` | [@metalmatze](https://github.com/metalmatze)| Red Hat | -| Lucas Servén Marín | lserven@gmail.com | `@squat` | [@squat](https://github.com/squat) | Red Hat | +| Lucas Servén Marín | lserven@gmail.com | `@squat` | [@squat](https://github.com/squat) | Red Hat | We are bunch of people from different companies with various interests and skills. We are from different parts of Europe: Germany, Lithuania, Poland and UK. @@ -30,6 +30,7 @@ Full list of triage persons is displayed below: | Name | Slack | GitHub | |-----------------------|--------------------------|------------------------------------------------------------| | Adrien Fillon | `@Adrien F` | [@adrien-f](https://github.com/adrien-f) | +| Ben Ye | `@yeya24` | [@yeya24](https://github.com/yeya24) | | Martin Chodur | `@FUSAKLA` | [@fusakla](https://github.com/fusakla) | | Michael Dai | `@jojohappy` | [@jojohappy](https://github.com/jojohappy) | From 912e1015621da831660d016ad693498337337ff8 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Fri, 24 Jan 2020 08:15:54 -0500 Subject: [PATCH 194/257] add flag max query samples (#1369) Signed-off-by: yeya24 --- cmd/thanos/query.go | 12 ++++++++---- docs/components/query.md | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 9d3412db31..4d71537770 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -6,12 +6,13 @@ import ( "math" "net/http" "path" + "strconv" "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/route" @@ -67,6 +68,8 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { instantDefaultMaxSourceResolution := modelDuration(cmd.Flag("query.instant.default.max_source_resolution", "default value for max_source_resolution for instant queries. If not set, defaults to 0s only taking raw resolution into account. 1h can be a good value if you use instant queries over time ranges that incorporate times outside of your raw-retention.").Default("0s").Hidden()) + maxQuerySamples := cmd.Flag("query.max-samples", "Maximum number of samples a single query can load into memory.").Default(strconv.Itoa(math.MaxInt32)).Int() + selectorLabels := cmd.Flag("selector-label", "Query selector labels that will be exposed in info endpoint (repeated)."). PlaceHolder("=\"\"").Strings() @@ -158,6 +161,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { time.Duration(*unhealthyStoreTimeout), time.Duration(*instantDefaultMaxSourceResolution), component.Query, + *maxQuerySamples, ) } } @@ -198,6 +202,7 @@ func runQuery( unhealthyStoreTimeout time.Duration, instantDefaultMaxSourceResolution time.Duration, comp component.Component, + maxQuerySamples int, ) error { // TODO(bplotka in PR #513 review): Move arguments into struct. duplicatedStores := prometheus.NewCounter(prometheus.CounterOpts{ @@ -242,9 +247,8 @@ func runQuery( Logger: logger, Reg: reg, MaxConcurrent: maxConcurrentQueries, - // TODO(bwplotka): Expose this as a flag: https://github.com/thanos-io/thanos/issues/703. - MaxSamples: math.MaxInt32, - Timeout: queryTimeout, + MaxSamples: maxQuerySamples, + Timeout: queryTimeout, }, ) ) diff --git a/docs/components/query.md b/docs/components/query.md index 81c0265f58..537f9e0f5b 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -318,6 +318,9 @@ Flags: which data is deduplicated. Still you will be able to query without deduplication using 'dedup=false' parameter. + --query.max-samples=2147483647 + Maximum number of samples a single query can + load into memory. --selector-label=="" ... Query selector labels that will be exposed in info endpoint (repeated). From 860460caf074f873927e52f55cf6b8da9e6c9618 Mon Sep 17 00:00:00 2001 From: Rob Coward Date: Fri, 24 Jan 2020 13:16:15 +0000 Subject: [PATCH 195/257] Bumped minio-go library to v6.0.45 (#2033) * Bumped minio-go library to v6.0.45 Signed-off-by: Rob Coward * Updated CHANGELOG with PR details Signed-off-by: Rob Coward --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09776497e7..dac014e34d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Fixed +- [#2033](https://github.com/thanos-io/thanos/pull/2033) minio-go: Fixed Issue #1494 support Web Identity providers for IAM credentials for AWS EKS - [#1985](https://github.com/thanos-io/thanos/pull/1985) store gateway: Fixed case where series entry is larger than 64KB in index. ### Added diff --git a/go.mod b/go.mod index f401230235..beb3c75d70 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe // indirect github.com/mattn/go-runewidth v0.0.6 // indirect github.com/miekg/dns v1.1.22 - github.com/minio/minio-go/v6 v6.0.44 + github.com/minio/minio-go/v6 v6.0.45 github.com/mozillazg/go-cos v0.13.0 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/oklog/run v1.0.0 diff --git a/go.sum b/go.sum index a101015a5a..a4ddc0f20a 100644 --- a/go.sum +++ b/go.sum @@ -405,8 +405,8 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc= github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/minio/minio-go/v6 v6.0.44 h1:CVwVXw+uCOcyMi7GvcOhxE8WgV+Xj8Vkf2jItDf/EGI= -github.com/minio/minio-go/v6 v6.0.44/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/minio-go/v6 v6.0.45 h1:aY4NI/DOgSbZiwGN3fEF4NAkC9An4bhaIWuJrQrRYew= +github.com/minio/minio-go/v6 v6.0.45/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= From d6bee65e72cc8d3822f0d16258348fb7a67505a9 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Sat, 25 Jan 2020 10:40:40 +0100 Subject: [PATCH 196/257] mixin/thanos: add DNS alerts for Thanos Ruler (#2027) Signed-off-by: Simon Pasquier --- examples/alerts/alerts.md | 28 ++++++++++++++++++++++ examples/alerts/alerts.yaml | 28 ++++++++++++++++++++++ mixin/thanos/alerts/ruler.libsonnet | 36 +++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index d603e751b2..dad259a3d0 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -142,6 +142,34 @@ rules: for: 5m labels: severity: warning +- alert: ThanosRulerQueryHighDNSFailures + annotations: + message: Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing + DNS queries for query endpoints. + expr: | + ( + sum by (job) (rate(thanos_ruler_query_apis_dns_failures_total{job=~"thanos-ruler.*"}[5m])) + / + sum by (job) (rate(thanos_ruler_query_apis_dns_lookups_total{job=~"thanos-ruler.*"}[5m])) + * 100 > 1 + ) + for: 15m + labels: + severity: warning +- alert: ThanosRulerAlertmanagerHighDNSFailures + annotations: + message: Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing + DNS queries for Alertmanager endpoints. + expr: | + ( + sum by (job) (rate(thanos_ruler_alertmanagers_dns_failures_total{job=~"thanos-ruler.*"}[5m])) + / + sum by (job) (rate(thanos_ruler_alertmanagers_dns_lookups_total{job=~"thanos-ruler.*"}[5m])) + * 100 > 1 + ) + for: 15m + labels: + severity: warning ``` ## Store Gateway diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index 434d741d58..5011e882b4 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -354,6 +354,34 @@ groups: for: 5m labels: severity: warning + - alert: ThanosRulerQueryHighDNSFailures + annotations: + message: Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing + DNS queries for query endpoints. + expr: | + ( + sum by (job) (rate(thanos_ruler_query_apis_dns_failures_total{job=~"thanos-ruler.*"}[5m])) + / + sum by (job) (rate(thanos_ruler_query_apis_dns_lookups_total{job=~"thanos-ruler.*"}[5m])) + * 100 > 1 + ) + for: 15m + labels: + severity: warning + - alert: ThanosRulerAlertmanagerHighDNSFailures + annotations: + message: Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing + DNS queries for Alertmanager endpoints. + expr: | + ( + sum by (job) (rate(thanos_ruler_alertmanagers_dns_failures_total{job=~"thanos-ruler.*"}[5m])) + / + sum by (job) (rate(thanos_ruler_alertmanagers_dns_lookups_total{job=~"thanos-ruler.*"}[5m])) + * 100 > 1 + ) + for: 15m + labels: + severity: warning - name: thanos-component-absent.rules rules: - alert: ThanosCompactorIsDown diff --git a/mixin/thanos/alerts/ruler.libsonnet b/mixin/thanos/alerts/ruler.libsonnet index 16d5c4e486..83f4d59807 100644 --- a/mixin/thanos/alerts/ruler.libsonnet +++ b/mixin/thanos/alerts/ruler.libsonnet @@ -114,6 +114,42 @@ severity: 'warning', }, }, + { + alert: 'ThanosRulerQueryHighDNSFailures', + annotations: { + message: 'Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for query endpoints.', + }, + expr: ||| + ( + sum by (job) (rate(thanos_ruler_query_apis_dns_failures_total{%(selector)s}[5m])) + / + sum by (job) (rate(thanos_ruler_query_apis_dns_lookups_total{%(selector)s}[5m])) + * 100 > 1 + ) + ||| % thanos.ruler, + 'for': '15m', + labels: { + severity: 'warning', + }, + }, + { + alert: 'ThanosRulerAlertmanagerHighDNSFailures', + annotations: { + message: 'Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for Alertmanager endpoints.', + }, + expr: ||| + ( + sum by (job) (rate(thanos_ruler_alertmanagers_dns_failures_total{%(selector)s}[5m])) + / + sum by (job) (rate(thanos_ruler_alertmanagers_dns_lookups_total{%(selector)s}[5m])) + * 100 > 1 + ) + ||| % thanos.ruler, + 'for': '15m', + labels: { + severity: 'warning', + }, + }, ], }, ], From cf4e4500588ac4f5874e8969a39d37e315d4176a Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sat, 25 Jan 2020 04:56:15 -0500 Subject: [PATCH 197/257] fix ruler ui alerts click (#2040) Signed-off-by: yeya24 --- pkg/ui/bindata.go | 140 +++++++++++++++++------------------ pkg/ui/templates/alerts.html | 2 +- 2 files changed, 71 insertions(+), 71 deletions(-) diff --git a/pkg/ui/bindata.go b/pkg/ui/bindata.go index d0d47c5c5f..22e167e463 100644 --- a/pkg/ui/bindata.go +++ b/pkg/ui/bindata.go @@ -160,12 +160,12 @@ func pkgUiTemplates_baseHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/_base.html", size: 1478, mode: os.FileMode(420), modTime: time.Unix(1575924708, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/_base.html", size: 1478, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _pkgUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x58\x5b\x6f\xdb\x36\x14\x7e\xcf\xaf\x38\x50\xfb\xb0\x02\x93\x85\x2c\xed\xc3\x1c\xd9\x43\x50\xa0\xdb\x80\xb6\x2b\x9a\xac\xaf\x01\x45\x1e\x5b\x6c\x18\x92\x20\x69\xc7\x99\xa6\xff\x3e\x90\x94\x7c\x51\xe5\x9b\x0a\x6c\x73\x00\x41\x22\x79\x2e\xdf\xb9\x33\x55\xc5\x70\xc6\x25\x42\x52\x22\x61\x49\x5d\x5f\x00\x00\xe4\x82\xcb\x07\x70\xcf\x1a\x27\x89\xc3\x95\xcb\xa8\xb5\x09\x18\x14\x93\xc4\xba\x67\x81\xb6\x44\x74\x09\x94\x06\x67\x93\xa4\xaa\x40\x13\x57\x7e\x32\x38\xe3\x2b\xa8\xeb\xcc\x3a\xe2\x38\xf5\x34\x19\x11\x68\x9c\x1d\x51\x6b\x7f\x59\x4e\xaa\x0a\x8a\x05\x17\xec\x0b\x1a\xcb\x95\x84\xba\x4e\xa6\x51\x9c\xa5\x86\x6b\x07\xd6\xd0\xfd\xec\xbe\xae\xb9\x7d\xdd\xc7\x2c\xcf\x22\xa3\xe9\x45\x55\xa1\x64\x75\x7d\x71\xb1\xc1\x47\x95\x74\x28\xdd\x1a\x22\xe3\x4b\xa0\x82\x58\x3b\x09\x5b\x84\x4b\x34\xe9\x4c\x2c\x38\x6b\xb4\x0a\xa7\xca\xcb\xe9\x4d\x90\x9a\x67\xe5\xe5\xd6\xba\xa7\xe6\x6c\x92\x04\x95\xde\x71\xe1\xd0\xd8\xa4\xe5\x57\x38\x99\xce\x8d\x5a\x68\x58\xbf\xa5\x4e\xcd\xe7\x02\x13\x60\xc4\x91\xe6\x63\x92\x14\x0b\xe7\x94\xb4\x5b\x02\xa3\xf5\x49\x81\x62\x8b\x59\x60\xa3\x0d\x7f\x24\xe6\x19\x08\x75\x7c\x89\x1d\x92\x40\xc6\xa5\x5e\xb8\xc6\x6b\xb4\x44\xfa\x50\xa8\x55\x02\x92\x3c\x62\x57\x4f\xaf\x39\x97\x91\x53\x84\x97\x00\x59\x38\x45\xd5\xa3\x16\xe8\x70\x92\xa8\xd9\x2c\x99\xc2\xef\xcd\x19\xf8\xa1\xaa\x60\xf4\x56\x2d\xa4\xb3\xa3\xf5\x62\x5d\xbf\xda\xd5\x3b\x0b\x8a\xff\x17\x60\x34\x4a\xc6\xe5\xfc\x10\x96\x4f\xf1\xc8\x0e\x94\x76\xed\x7f\x84\x64\xc6\xcd\x11\x20\xef\xc2\x89\x1d\x1c\xcd\xd2\x69\x30\xb2\xc2\x6c\x05\x72\xc6\xf8\xb2\x13\xd7\x0d\x42\x5b\xaa\xa7\x94\x48\xa9\x7c\xfe\xf5\x44\x29\x6f\x0f\xce\xc5\xb3\x2e\x39\x55\x12\xd6\x6f\xe9\x42\x06\xa8\xc8\x7c\x56\xf2\x0e\x65\x0c\xfb\xc6\x24\xf1\x23\xd9\x2b\x15\x1c\x77\x3e\x57\xfc\x06\xec\xa8\x73\xdb\x59\xc9\xb3\xc8\x6b\x3f\x3a\x47\x0a\x81\x9b\xbc\xb5\x77\xfe\x7b\x2d\x3a\xee\x86\x67\x5a\x28\xc3\xd0\x20\x6b\x3e\xa9\x12\x82\x68\x8b\xac\x6b\x04\x57\x28\xf6\xbc\xbb\x56\x55\x2f\x03\xf7\x5b\x47\x1c\xde\xa9\xcf\xea\xe9\xad\xe7\x0f\xe3\x09\x8c\x6e\x7a\x36\x9a\x82\xb4\x21\x37\x44\xce\x11\x46\xbf\xfa\xba\xd1\xdd\x8d\x42\xcd\xb7\x61\x16\x37\x18\x84\xfa\x3c\x49\x34\x61\x3e\xb2\xc7\xf0\x93\x5e\xf5\x04\xe5\x46\xd8\xe8\x0f\xc3\xe7\x5c\x12\xf1\x8e\x0b\xac\x6b\x98\xfa\xb5\x8f\xe4\x11\x7b\x04\x47\x93\x3a\xd6\x13\xe4\x59\x9f\x4e\x6b\x28\x01\x36\x97\xf3\xcf\x0b\x81\x7d\x88\x5a\xb3\x6d\x15\xa4\x68\xae\xad\x85\x7d\xfa\x38\xd3\xfa\x2f\x58\x1d\xc2\x33\xad\x2a\x2e\x19\xae\xa0\xdf\x15\xa3\xb0\x50\xd7\xf1\xf0\xbd\x6f\x7d\x68\x0e\x98\x29\x77\x6c\xba\x89\xf7\x10\xe0\xb4\xc4\xa5\x51\x32\x65\xea\x49\xc6\x18\x87\xbc\x98\xae\x6d\x97\x67\xc5\xd4\xa7\xa8\x40\x09\x3b\xb8\xbc\xd0\xf0\xf9\xaa\xdf\x92\xfb\xad\xd9\x87\xf6\x9e\xa1\x23\x5c\x74\x73\xb3\xab\xfb\xde\x4d\x68\x72\xfe\xf0\x89\x70\x4a\x1b\x6c\x83\x8b\x71\xab\x05\x79\x1e\x17\x42\xd1\x87\x6b\x68\x63\xed\xe7\xd1\x1b\xbd\xba\x86\x99\x92\x2e\xb5\xfc\x2f\x1c\x5f\x5e\xf9\x6f\xaa\x84\x32\xe3\x17\x57\x57\x57\xd7\xf0\xa4\x0c\x4b\x0b\x83\xe4\x61\x1c\x9e\x29\x11\xe2\x1a\x0a\x42\x1f\x7c\x97\x94\x2c\x6d\x0e\xcf\xde\xf8\xbf\x6b\x88\x69\x38\xbe\xd4\x2b\xb0\x4a\x70\x06\x2f\x28\xa5\xed\x72\x6a\x08\xe3\x0b\x3b\x7e\xad\x57\xd7\x09\x4c\x73\xaa\x18\x7a\x0f\xfc\x76\xf7\xe1\xfd\xad\xe4\x5a\xa3\xdb\x1a\x25\xbc\x4f\xc2\x89\x3c\xd3\x06\x8f\x98\x24\x3b\x6a\x93\xaa\xe2\xb3\xae\x67\x8f\x1b\x31\x96\x98\xe3\xf5\xa6\x54\x4b\x34\xcd\xbb\x7d\x6c\xc2\x14\x05\x3e\xa2\x74\xf6\x3e\xac\x1f\x70\xf9\xae\xc8\x75\xc4\x9c\x48\x11\xa9\xca\xe9\x7b\xdf\x3b\x6c\x9e\xb9\xf2\x3c\xba\x90\x5b\xe7\x93\xc5\x5c\x87\x5b\x2e\xe9\x00\xea\x2f\x44\x2c\xce\x20\xdb\x9f\x61\xdd\x5f\x5b\xc3\xce\x75\xf5\x46\xb9\x13\x05\x6d\x08\x8e\xe4\xeb\x41\x2d\x43\xc3\xff\x11\x5e\x2e\xbd\x3d\x42\x0d\x8d\x6e\x1c\x7d\x20\xfa\x0c\xad\x77\x14\xb2\x9a\xc8\xf5\xd8\x43\xd8\x1c\x21\x3c\xdb\xd1\x27\x99\x56\x55\x94\x5b\xd7\x7e\x7c\x8f\xb2\xeb\x3a\xc9\x33\x4f\x39\x04\x4c\x1c\xde\xcf\x52\x72\x6f\x31\xdd\x4b\xe1\x6b\xfa\x36\xb4\xdd\xf6\xd1\xf4\x08\xf8\x1b\xb6\x3b\x48\x6c\x1f\x75\x0d\xfe\x52\x82\xf7\x5c\x32\x4e\x89\x53\x06\xfc\x2d\x29\x5d\x68\x8d\x86\x12\x8b\xde\x24\x6d\x8f\x69\xac\x30\x4c\xc1\xaa\x6a\x7b\xa0\x1b\xfd\x79\xf7\xd6\x73\x1b\xc8\xe6\x4b\x74\xca\x79\xf4\xa7\xa7\x09\x04\xbf\x01\x9f\xc1\xe8\x66\x33\x91\x0d\x88\x3a\x5f\xaf\x3a\x6d\x46\x2a\xb9\x19\xd2\x62\x29\xdc\x3f\x96\x9e\x26\xa3\xf4\x5d\xc9\xfb\x65\x92\xbc\x4e\xa6\x37\xdb\x33\xe4\x39\x95\x07\xce\xb6\xd1\xbf\x07\x91\xed\x40\x1c\x96\xf9\x4c\x0c\x23\x84\xc3\x25\xe9\xfb\x22\xe4\x5b\x2d\xdd\x56\x05\xca\x33\xe6\x86\x2b\x1d\xf9\xf9\x84\x69\xab\x58\x9e\xb1\x01\xf5\xb8\xfd\x0d\x29\x65\x6b\x3d\xb2\x21\xe6\x1f\x50\x20\xce\x4c\xf2\xd3\x11\x9d\x7a\x36\xcf\xc2\x44\x73\x6c\xe0\x3a\xcc\xec\xdc\x71\xba\x8f\x5f\x55\xa1\xb0\x7d\x37\x9f\x43\x57\xae\xfd\x6a\x7f\x54\xb1\x79\xf8\xbb\xb9\xf1\x57\x1f\x88\xff\x8c\x62\xdf\x75\xb3\xea\x53\x3c\xcf\x3a\x17\xd1\x1d\x93\x36\xe3\x6c\x4b\xf8\x4f\x00\x00\x00\xff\xff\xe3\xad\x3c\xbb\xf4\x13\x00\x00") +var _pkgUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x58\x5b\x6f\xdb\x36\x14\x7e\xcf\xaf\x38\x50\xfb\xb0\x02\x93\x85\x2c\xed\xc3\x1c\xd9\x43\x50\xa0\xdb\x80\xb6\x2b\x9a\x34\xaf\x01\x25\x1e\x5b\x6c\x18\x92\x23\x29\xc7\x9e\xa6\xff\x3e\x90\x94\x7c\x51\xe5\x9b\x0a\x6c\x73\x00\x41\x22\x79\x2e\xdf\xb9\x33\x55\x45\x71\xc6\x04\x42\x54\x20\xa1\x51\x5d\x5f\x00\x00\xa4\x9c\x89\x47\xb0\x2b\x85\x93\xc8\xe2\xd2\x26\xb9\x31\x11\x68\xe4\x93\xc8\xd8\x15\x47\x53\x20\xda\x08\x0a\x8d\xb3\x49\x54\x55\xa0\x88\x2d\x3e\x69\x9c\xb1\x25\xd4\x75\x62\x2c\xb1\x2c\x77\x34\x09\xe1\xa8\xad\x19\xe5\xc6\xfc\xb2\x98\x54\x15\x64\x25\xe3\xf4\x1e\xb5\x61\x52\x40\x5d\x47\xd3\x20\xce\xe4\x9a\x29\x0b\x46\xe7\xfb\xd9\x7d\x5d\x73\xfb\xba\x8f\x59\x9a\x04\x46\xd3\x8b\xaa\x42\x41\xeb\xfa\xe2\x62\x83\x2f\x97\xc2\xa2\xb0\x6b\x88\x94\x2d\x20\xe7\xc4\x98\x89\xdf\x22\x4c\xa0\x8e\x67\xbc\x64\xb4\xd1\xca\x9f\x2a\x2e\xa7\x37\x5e\x6a\x9a\x14\x97\x5b\xeb\x8e\x9a\xd1\x49\xe4\x55\x7a\xc7\xb8\x45\x6d\xa2\x96\x5f\x66\x45\x3c\xd7\xb2\x54\xb0\x7e\x8b\xad\x9c\xcf\x39\x46\x40\x89\x25\xcd\xc7\x24\xca\x4a\x6b\xa5\x30\x5b\x02\x83\xf5\x49\x86\x7c\x8b\x99\x67\xa3\x34\x7b\x22\x7a\x05\x24\xb7\x6c\x81\x1d\x12\x4f\xc6\x84\x2a\x6d\xe3\xb5\xbc\xc0\xfc\x31\x93\xcb\x08\x04\x79\xc2\xae\x9e\x4e\x73\x26\x02\xa7\x00\x2f\x02\x52\x5a\x99\xcb\x27\xc5\xd1\xe2\x24\x92\xb3\x59\x34\x85\xdf\x9b\x33\xf0\x43\x55\xc1\xe8\xad\x2c\x85\x35\xa3\xf5\x62\x5d\xbf\xda\xd5\x3b\xf1\x8a\xff\x17\x60\x14\x0a\xca\xc4\xfc\x10\x96\x4f\xe1\xc8\x0e\x94\x76\xed\x7f\x84\x64\xc6\xf4\x11\x20\xef\xfc\x89\x1d\x1c\xcd\xd2\x69\x30\x92\x4c\x6f\x05\x72\x42\xd9\xa2\x13\xd7\x0d\x42\x53\xc8\xe7\x98\x08\x21\x5d\xfe\xf5\x44\x29\x6b\x0f\xce\xf9\x4a\x15\x2c\x97\x02\xd6\x6f\x71\x29\x3c\x54\xa4\x2e\x2b\x59\x87\x32\x84\x7d\x63\x92\xf0\x11\xed\x95\x0a\x96\x59\x97\x2b\x6e\x03\x76\xd4\xb9\xed\xac\xa4\x49\xe0\xb5\x1f\x9d\x25\x19\xc7\x4d\xde\x9a\x3b\xf7\xbd\x16\x1d\x76\xfd\x33\xce\xa4\xa6\xa8\x91\x36\x9f\xb9\xe4\x9c\x28\x83\xb4\x6b\x04\x9b\x49\xba\xda\x5d\xab\xaa\x97\x9e\xfb\xad\x25\x16\xef\xe4\x67\xf9\xfc\xd6\xf1\x87\xf1\x04\x46\x37\x3d\x1b\x4d\x41\xda\x90\x6b\x22\xe6\x08\xa3\x5f\x5d\xdd\xe8\xee\x06\xa1\xfa\xdb\x30\x0b\x1b\x14\x7c\x7d\x9e\x44\x8a\x50\x17\xd9\x63\xf8\x49\x2d\x7b\x82\x72\x23\x6c\xf4\x87\x66\x73\x26\x08\x7f\xc7\x38\xd6\x35\x4c\xdd\xda\x47\xf2\x84\x3d\x82\x83\x49\x2d\xed\x09\xf2\xa4\x4f\xa7\x35\x14\x0f\x9b\x89\xf9\xe7\x92\x63\x1f\xa2\xd6\x6c\x5b\x05\x29\x98\x6b\x6b\x61\x9f\x3e\x56\xb7\xfe\xf3\x56\x07\xff\x8c\xab\x8a\x09\x8a\x4b\xe8\x77\xc5\xc8\x2f\xd4\x75\x38\xfc\xe0\x5a\x1f\xea\x03\x66\x4a\x2d\x9d\x6e\xe2\xdd\x07\x78\x5e\xe0\x42\x4b\x11\x53\xf9\x2c\x42\x8c\x43\x9a\x4d\xd7\xb6\x4b\x93\x6c\xea\x52\x94\xa3\x80\x1d\x5c\x4e\xa8\xff\x7c\xd5\x6f\xc9\xfd\xd6\xec\x43\xfb\x40\xd1\x12\xc6\xbb\xb9\xd9\xd5\x7d\xef\x26\x34\x39\x7f\xf8\x84\x3f\xa5\x34\xb6\xc1\x45\x99\x51\x9c\xac\xc6\x19\x97\xf9\xe3\x35\xb4\xb1\xf6\xf3\xe8\x8d\x5a\x5e\xc3\x4c\x0a\x1b\x1b\xf6\x17\x8e\x2f\xaf\xdc\x77\x2e\xb9\xd4\xe3\x17\x57\x57\x57\xd7\xf0\x2c\x35\x8d\x33\x8d\xe4\x71\xec\x9f\x31\xe1\xfc\x1a\x32\x92\x3f\xba\x2e\x29\x68\xdc\x1c\x9e\xbd\x71\x7f\xd7\x10\xd2\x70\x7c\xa9\x96\x60\x24\x67\x14\x5e\xe4\x79\xde\x2e\xc7\x9a\x50\x56\x9a\xf1\x6b\xb5\xbc\x8e\x60\x9a\xe6\x92\xa2\xf3\xc0\x6f\x77\x1f\xde\xdf\x0a\xa6\x14\x5a\xf8\xb3\x44\xbd\xfa\xf2\xf9\xbd\xf3\x88\xdf\x4f\x13\xa5\xf1\x88\x41\x92\xa3\x16\xa9\x2a\x36\xeb\xfa\xf5\xb8\x09\x43\x81\x39\x5e\x6d\x0a\xb9\x40\xdd\xbc\x9b\xa7\x26\x48\x91\xe3\x13\x0a\x6b\x1e\xfc\xfa\x01\x87\xef\x8a\x5c\xc7\xcb\x89\x14\x81\xaa\x98\xbe\x77\x9d\xc3\xa4\x89\x2d\xce\xa3\xf3\x99\x75\x3e\x59\xc8\x74\xb8\x65\x22\x1f\x40\x7d\x4f\x78\x79\x06\xd9\xfe\xfc\xea\xfe\xda\x0a\x76\xae\xab\x37\xca\x9d\x28\x68\x43\x70\x24\x5b\x0f\x6a\xe9\xdb\xfd\x8f\xf0\x72\xe1\xec\xe1\x2b\x68\x70\xe3\xe8\x03\x51\x67\x68\xbd\xa3\x90\x51\x44\xac\x87\x1e\x42\xe7\x08\xfe\xd9\x0e\x3e\xd1\xb4\xaa\x82\xdc\xba\x76\xc3\x7b\x90\x5d\xd7\x51\x9a\x38\xca\x21\x60\xc2\xe8\x7e\x96\x92\x7b\x4b\xe9\x5e\x0a\x57\xd1\xb7\xa1\xed\x36\x8f\xa6\x43\xc0\xdf\xb0\xdd\x3f\x42\xf3\xa8\x6b\x70\x57\x12\x7c\x60\x82\xb2\x9c\x58\xa9\xc1\xdd\x91\xe2\x52\x29\xd4\x39\x31\xe8\x4c\xd2\x76\x98\xc6\x0a\xc3\x14\xac\xaa\xb6\x03\xda\xd1\x97\xbb\xb7\x8e\xdb\x40\x36\xf7\xc1\x29\xe7\xd1\x9f\x9e\x26\xe0\xfd\x06\x6c\x06\xa3\x9b\xcd\x3c\x36\x20\xea\x5c\xbd\xea\x34\x19\x21\xc5\x66\x44\x0b\xa5\x70\xff\x50\x7a\x9a\x8c\xc2\xf5\x24\xe7\x97\x49\xf4\x3a\x9a\xde\x6c\x4f\x90\xe7\x54\x1e\x38\xdb\x46\xff\x1e\x44\xba\x03\x71\x58\xe6\x53\x3e\x8c\x10\x0e\x97\xa4\xef\x8b\x90\x6f\xb5\xb4\x5b\x15\x28\x4d\xa8\x1d\xae\x74\xe0\xe7\x12\xa6\xad\x62\x69\x42\x07\xd4\xe3\xf6\x37\xa4\x94\xad\xf5\x48\x86\x98\x7f\x40\x81\x38\x33\xc9\x4f\x47\x74\xea\xd9\x34\xf1\x13\xcd\xb1\x81\xeb\x30\xb3\x73\x87\xe9\x3e\x7e\x55\x85\xdc\xf4\xdd\x7b\x0e\x5d\xb8\xf6\xab\xfd\x51\x86\xe6\xe1\x6e\xe6\xda\x5d\x7c\x20\xfc\x2b\x8a\x7e\xd7\xbd\xaa\x4f\xf1\x34\xe9\x5c\x43\x77\x4c\xda\x8c\xb3\x2d\xe1\x3f\x01\x00\x00\xff\xff\x78\x1e\x88\x4d\xf2\x13\x00\x00") func pkgUiTemplatesAlertsHtmlBytes() ([]byte, error) { return bindataRead( @@ -180,7 +180,7 @@ func pkgUiTemplatesAlertsHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 5108, mode: os.FileMode(420), modTime: time.Unix(1575983680, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/alerts.html", size: 5106, mode: os.FileMode(436), modTime: time.Unix(1579916569, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -200,7 +200,7 @@ func pkgUiTemplatesBucketHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket.html", size: 1281, mode: os.FileMode(420), modTime: time.Unix(1564066864, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket.html", size: 1281, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -220,7 +220,7 @@ func pkgUiTemplatesBucket_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 787, mode: os.FileMode(420), modTime: time.Unix(1576072884, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/bucket_menu.html", size: 787, mode: os.FileMode(436), modTime: time.Unix(1579791415, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -240,7 +240,7 @@ func pkgUiTemplatesGraphHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/graph.html", size: 2298, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/graph.html", size: 2298, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -260,7 +260,7 @@ func pkgUiTemplatesQuery_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1515, mode: os.FileMode(420), modTime: time.Unix(1576072716, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/query_menu.html", size: 1515, mode: os.FileMode(436), modTime: time.Unix(1579796928, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -280,7 +280,7 @@ func pkgUiTemplatesRule_menuHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 963, mode: os.FileMode(420), modTime: time.Unix(1576025253, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rule_menu.html", size: 963, mode: os.FileMode(436), modTime: time.Unix(1579791415, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -300,7 +300,7 @@ func pkgUiTemplatesRulesHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/rules.html", size: 1944, mode: os.FileMode(420), modTime: time.Unix(1575919007, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/rules.html", size: 1944, mode: os.FileMode(436), modTime: time.Unix(1579791415, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -320,7 +320,7 @@ func pkgUiTemplatesStatusHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/status.html", size: 1272, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/status.html", size: 1272, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -340,7 +340,7 @@ func pkgUiTemplatesStoresHtml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/templates/stores.html", size: 2347, mode: os.FileMode(420), modTime: time.Unix(1574377450, 0)} + info := bindataFileInfo{name: "pkg/ui/templates/stores.html", size: 2347, mode: os.FileMode(436), modTime: time.Unix(1579791415, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -360,7 +360,7 @@ func pkgUiStaticCssAlertsCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/alerts.css", size: 401, mode: os.FileMode(420), modTime: time.Unix(1575919484, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/alerts.css", size: 401, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -380,7 +380,7 @@ func pkgUiStaticCssGraphCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/graph.css", size: 3844, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/graph.css", size: 3844, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -400,7 +400,7 @@ func pkgUiStaticCssPrometheusCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/prometheus.css", size: 470, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/prometheus.css", size: 470, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -420,7 +420,7 @@ func pkgUiStaticCssRulesCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/css/rules.css", size: 195, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/css/rules.css", size: 195, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -440,7 +440,7 @@ func pkgUiStaticImgAjaxLoaderGif() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/img/ajax-loader.gif", size: 847, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -460,7 +460,7 @@ func pkgUiStaticImgFaviconIco() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/img/favicon.ico", size: 15886, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/img/favicon.ico", size: 15886, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -480,7 +480,7 @@ func pkgUiStaticJsAlertsJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 2637, mode: os.FileMode(420), modTime: time.Unix(1575983680, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/alerts.js", size: 2637, mode: os.FileMode(436), modTime: time.Unix(1579791415, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -500,7 +500,7 @@ func pkgUiStaticJsBucketJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/bucket.js", size: 2834, mode: os.FileMode(420), modTime: time.Unix(1562774024, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/bucket.js", size: 2834, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -520,7 +520,7 @@ func pkgUiStaticJsGraphJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 38722, mode: os.FileMode(420), modTime: time.Unix(1576074054, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/graph.js", size: 38722, mode: os.FileMode(436), modTime: time.Unix(1579796928, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -540,7 +540,7 @@ func pkgUiStaticJsGraph_templateHandlebar() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/js/graph_template.handlebar", size: 8984, mode: os.FileMode(420), modTime: time.Unix(1572558524, 0)} + info := bindataFileInfo{name: "pkg/ui/static/js/graph_template.handlebar", size: 8984, mode: os.FileMode(436), modTime: time.Unix(1579791415, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -560,7 +560,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapGridCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.css", size: 37644, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.css", size: 37644, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -580,7 +580,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapGridMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.min.css", size: 28977, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-grid.min.css", size: 28977, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -600,7 +600,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapRebootCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.css", size: 4896, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.css", size: 4896, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -620,7 +620,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapRebootMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.min.css", size: 4019, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap-reboot.min.css", size: 4019, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -640,7 +640,7 @@ func pkgUiStaticVendorBootstrap413CssBootstrapMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap.min.css", size: 140936, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/css/bootstrap.min.css", size: 140936, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -660,7 +660,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapBundleJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.js", size: 212345, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.js", size: 212345, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -680,7 +680,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapBundleMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.min.js", size: 70966, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.bundle.min.js", size: 70966, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -700,7 +700,7 @@ func pkgUiStaticVendorBootstrap413JsBootstrapMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.min.js", size: 51039, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap-4.1.3/js/bootstrap.min.js", size: 51039, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -720,7 +720,7 @@ func pkgUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs() (*asset, err return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 11273, mode: os.FileMode(420), modTime: time.Unix(1569428366, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js", size: 11273, mode: os.FileMode(436), modTime: time.Unix(1579791415, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -740,7 +740,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsCssBootstrapGlyphiconsCss() (*asset, e return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.css", size: 14523, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.css", size: 14523, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -760,7 +760,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsCssBootstrapGlyphiconsMinCss() (*asset return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css", size: 11830, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css", size: 11830, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -780,7 +780,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Eot() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.eot", size: 98620, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.eot", size: 98620, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -800,7 +800,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Svg() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.svg", size: 507478, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.svg", size: 507478, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -820,7 +820,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Ttf() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.ttf", size: 98384, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.ttf", size: 98384, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -840,7 +840,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Woff() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff", size: 63712, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff", size: 63712, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -860,7 +860,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaBrands400Woff2() (*a return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff2", size: 54420, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-brands-400.woff2", size: 54420, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -880,7 +880,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Eot() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.eot", size: 31156, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.eot", size: 31156, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -900,7 +900,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Svg() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.svg", size: 107199, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.svg", size: 107199, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -920,7 +920,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Ttf() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.ttf", size: 30928, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.ttf", size: 30928, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -940,7 +940,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Woff() (*a return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff", size: 14712, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff", size: 14712, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -960,7 +960,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaRegular400Woff2() (* return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff2", size: 12220, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-regular-400.woff2", size: 12220, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -980,7 +980,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Eot() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.eot", size: 102152, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.eot", size: 102152, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1000,7 +1000,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Svg() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.svg", size: 378215, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.svg", size: 378215, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1020,7 +1020,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Ttf() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.ttf", size: 101932, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.ttf", size: 101932, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1040,7 +1040,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Woff() (*ass return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff", size: 48704, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff", size: 48704, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1060,7 +1060,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsFontawesomeFaSolid900Woff2() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff2", size: 38784, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/fontawesome/fa-solid-900.woff2", size: 38784, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1080,7 +1080,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.eot", size: 20127, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.eot", size: 20127, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1100,7 +1100,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.svg", size: 108738, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.svg", size: 108738, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1120,7 +1120,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.ttf", size: 45404, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.ttf", size: 45404, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1140,7 +1140,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff", size: 23424, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff", size: 23424, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1160,7 +1160,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsFontsGlyphiconsGlyphiconsHalflingsRegu return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff2", size: 18028, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/fonts/glyphicons/glyphicons-halflings-regular.woff2", size: 18028, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1180,7 +1180,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeCss() (*asset return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.css", size: 51062, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.css", size: 51062, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1200,7 +1200,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeLess() (*asse return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.less", size: 53867, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.less", size: 53867, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1220,7 +1220,7 @@ func pkgUiStaticVendorBootstrap4GlyphiconsMapsGlyphiconsFontawesomeMinCss() (*as return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.min.css", size: 42307, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/bootstrap4-glyphicons/maps/glyphicons-fontawesome.min.css", size: 42307, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1240,7 +1240,7 @@ func pkgUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinC return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.css", size: 7771, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1260,7 +1260,7 @@ func pkgUiStaticVendorEonasdanBootstrapDatetimepickerBootstrapDatetimepickerMinJ return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/eonasdan-bootstrap-datetimepicker/bootstrap-datetimepicker.min.js", size: 48881, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1280,7 +1280,7 @@ func pkgUiStaticVendorFuzzyFuzzyJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/fuzzy/fuzzy.js", size: 5669, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/fuzzy/fuzzy.js", size: 5669, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1300,7 +1300,7 @@ func pkgUiStaticVendorJsJquery331MinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery-3.3.1.min.js", size: 86927, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery-3.3.1.min.js", size: 86927, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1320,7 +1320,7 @@ func pkgUiStaticVendorJsJqueryHotkeysJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.hotkeys.js", size: 4490, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.hotkeys.js", size: 4490, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1340,7 +1340,7 @@ func pkgUiStaticVendorJsJqueryMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.min.js", size: 86671, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.min.js", size: 86671, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1360,7 +1360,7 @@ func pkgUiStaticVendorJsJquerySelectionJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.selection.js", size: 12881, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/jquery.selection.js", size: 12881, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1380,7 +1380,7 @@ func pkgUiStaticVendorJsPopperMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/js/popper.min.js", size: 19236, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/js/popper.min.js", size: 19236, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1400,7 +1400,7 @@ func pkgUiStaticVendorMomentMomentTimezoneWithDataMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment-timezone-with-data.min.js", size: 184495, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment-timezone-with-data.min.js", size: 184495, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1420,7 +1420,7 @@ func pkgUiStaticVendorMomentMomentMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment.min.js", size: 51825, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/moment/moment.min.js", size: 51825, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1440,7 +1440,7 @@ func pkgUiStaticVendorMustacheMustacheMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/mustache/mustache.min.js", size: 9528, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1460,7 +1460,7 @@ func pkgUiStaticVendorRickshawRickshawMinCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.css", size: 6102, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1480,7 +1480,7 @@ func pkgUiStaticVendorRickshawRickshawMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/rickshaw.min.js", size: 76322, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1500,7 +1500,7 @@ func pkgUiStaticVendorRickshawVendorD3LayoutMinJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.layout.min.js", size: 17514, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -1520,7 +1520,7 @@ func pkgUiStaticVendorRickshawVendorD3V3Js() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(420), modTime: time.Unix(1561294280, 0)} + info := bindataFileInfo{name: "pkg/ui/static/vendor/rickshaw/vendor/d3.v3.js", size: 144718, mode: os.FileMode(420), modTime: time.Unix(1576511495, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/pkg/ui/templates/alerts.html b/pkg/ui/templates/alerts.html index 21c52117ba..e805c469db 100644 --- a/pkg/ui/templates/alerts.html +++ b/pkg/ui/templates/alerts.html @@ -39,7 +39,7 @@

    Alerts

    -
    {{.HTMLSnippet pathPrefix}}
    +
    {{.HTMLSnippet queryURL}}
    {{if $activeAlerts}} From cedea118cb2ce38e652de623246ba61f531cde96 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Sun, 26 Jan 2020 08:58:22 +0100 Subject: [PATCH 198/257] Revert "add flag max query samples (#1369)" (#2043) This reverts commit 912e1015621da831660d016ad693498337337ff8. --- cmd/thanos/query.go | 12 ++++-------- docs/components/query.md | 3 --- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 4d71537770..9d3412db31 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -6,13 +6,12 @@ import ( "math" "net/http" "path" - "strconv" "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" - "github.com/opentracing/opentracing-go" + opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/route" @@ -68,8 +67,6 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { instantDefaultMaxSourceResolution := modelDuration(cmd.Flag("query.instant.default.max_source_resolution", "default value for max_source_resolution for instant queries. If not set, defaults to 0s only taking raw resolution into account. 1h can be a good value if you use instant queries over time ranges that incorporate times outside of your raw-retention.").Default("0s").Hidden()) - maxQuerySamples := cmd.Flag("query.max-samples", "Maximum number of samples a single query can load into memory.").Default(strconv.Itoa(math.MaxInt32)).Int() - selectorLabels := cmd.Flag("selector-label", "Query selector labels that will be exposed in info endpoint (repeated)."). PlaceHolder("=\"\"").Strings() @@ -161,7 +158,6 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { time.Duration(*unhealthyStoreTimeout), time.Duration(*instantDefaultMaxSourceResolution), component.Query, - *maxQuerySamples, ) } } @@ -202,7 +198,6 @@ func runQuery( unhealthyStoreTimeout time.Duration, instantDefaultMaxSourceResolution time.Duration, comp component.Component, - maxQuerySamples int, ) error { // TODO(bplotka in PR #513 review): Move arguments into struct. duplicatedStores := prometheus.NewCounter(prometheus.CounterOpts{ @@ -247,8 +242,9 @@ func runQuery( Logger: logger, Reg: reg, MaxConcurrent: maxConcurrentQueries, - MaxSamples: maxQuerySamples, - Timeout: queryTimeout, + // TODO(bwplotka): Expose this as a flag: https://github.com/thanos-io/thanos/issues/703. + MaxSamples: math.MaxInt32, + Timeout: queryTimeout, }, ) ) diff --git a/docs/components/query.md b/docs/components/query.md index 537f9e0f5b..81c0265f58 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -318,9 +318,6 @@ Flags: which data is deduplicated. Still you will be able to query without deduplication using 'dedup=false' parameter. - --query.max-samples=2147483647 - Maximum number of samples a single query can - load into memory. --selector-label=="" ... Query selector labels that will be exposed in info endpoint (repeated). From e1ae2bef4d77f99d72c2fd193cafca462fd3bfc8 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 26 Jan 2020 02:58:48 -0500 Subject: [PATCH 199/257] don't calculate compact group key multiple times (#2044) Signed-off-by: yeya24 --- pkg/compact/compact.go | 22 ++++++++++++---------- pkg/verifier/overlapped_blocks.go | 3 ++- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index b9f5332fd6..49d641d0b3 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -187,26 +187,27 @@ func (s *Syncer) Groups() (res []*Group, err error) { groups := map[string]*Group{} for _, m := range s.blocks { - g, ok := groups[GroupKey(m.Thanos)] + groupKey := GroupKey(m.Thanos) + g, ok := groups[groupKey] if !ok { g, err = newGroup( - log.With(s.logger, "compactionGroup", GroupKey(m.Thanos)), + log.With(s.logger, "compactionGroup", groupKey), s.bkt, labels.FromMap(m.Thanos.Labels), m.Thanos.Downsample.Resolution, s.acceptMalformedIndex, s.enableVerticalCompaction, - s.metrics.compactions.WithLabelValues(GroupKey(m.Thanos)), - s.metrics.compactionRunsStarted.WithLabelValues(GroupKey(m.Thanos)), - s.metrics.compactionRunsCompleted.WithLabelValues(GroupKey(m.Thanos)), - s.metrics.compactionFailures.WithLabelValues(GroupKey(m.Thanos)), - s.metrics.verticalCompactions.WithLabelValues(GroupKey(m.Thanos)), + s.metrics.compactions.WithLabelValues(groupKey), + s.metrics.compactionRunsStarted.WithLabelValues(groupKey), + s.metrics.compactionRunsCompleted.WithLabelValues(groupKey), + s.metrics.compactionFailures.WithLabelValues(groupKey), + s.metrics.verticalCompactions.WithLabelValues(groupKey), s.metrics.garbageCollectedBlocks, ) if err != nil { return nil, errors.Wrap(err, "create compaction group") } - groups[GroupKey(m.Thanos)] = g + groups[groupKey] = g res = append(res, g) } if err := g.Add(m); err != nil { @@ -687,8 +688,9 @@ func (cg *Group) compact(ctx context.Context, dir string, comp tsdb.Compactor) ( return false, ulid.ULID{}, errors.Wrapf(err, "read meta from %s", pdir) } - if cg.Key() != GroupKey(meta.Thanos) { - return false, ulid.ULID{}, halt(errors.Wrapf(err, "compact planned compaction for mixed groups. group: %s, planned block's group: %s", cg.Key(), GroupKey(meta.Thanos))) + cgKey, groupKey := cg.Key(), GroupKey(meta.Thanos) + if cgKey != groupKey { + return false, ulid.ULID{}, halt(errors.Wrapf(err, "compact planned compaction for mixed groups. group: %s, planned block's group: %s", cgKey, groupKey)) } for _, s := range meta.Compaction.Sources { diff --git a/pkg/verifier/overlapped_blocks.go b/pkg/verifier/overlapped_blocks.go index 4c35328e41..1835aa5b7d 100644 --- a/pkg/verifier/overlapped_blocks.go +++ b/pkg/verifier/overlapped_blocks.go @@ -53,7 +53,8 @@ func fetchOverlaps(ctx context.Context, logger log.Logger, bkt objstore.Bucket, groupMetasMap := map[string][]tsdb.BlockMeta{} for _, meta := range metas { - groupMetasMap[compact.GroupKey(meta.Thanos)] = append(groupMetasMap[compact.GroupKey(meta.Thanos)], meta.BlockMeta) + groupKey := compact.GroupKey(meta.Thanos) + groupMetasMap[groupKey] = append(groupMetasMap[groupKey], meta.BlockMeta) } overlaps := map[string]tsdb.Overlaps{} From 84b14c250c462606bce4615c61671f02fd54ad7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Mon, 27 Jan 2020 18:23:57 +0100 Subject: [PATCH 200/257] Improve help text for shipper metrics. (#2048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/shipper/shipper.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index f06b4478c0..00ced432db 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -39,7 +39,7 @@ func newMetrics(r prometheus.Registerer, uploadCompacted bool) *metrics { m.dirSyncs = prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_shipper_dir_syncs_total", - Help: "Total dir sync attempts", + Help: "Total number of dir syncs", }) m.dirSyncFailures = prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_shipper_dir_sync_failures_total", @@ -47,11 +47,11 @@ func newMetrics(r prometheus.Registerer, uploadCompacted bool) *metrics { }) m.uploads = prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_shipper_uploads_total", - Help: "Total object upload attempts", + Help: "Total number of uploaded blocks", }) m.uploadFailures = prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_shipper_upload_failures_total", - Help: "Total number of failed object uploads", + Help: "Total number of block upload failures", }) m.uploadedCompacted = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "thanos_shipper_upload_compacted_done", From 4a23b36a017961a2ab6b78c4ca585a680d922a0d Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Jan 2020 19:12:01 +0100 Subject: [PATCH 201/257] tracing: Support sampling on Elastic APM (#2049) * support sampling on Elastic APM tracing fixes thanos-io/thanos#2039 Signed-off-by: Igor Wiedler * add CHANGELOG entry Signed-off-by: Igor Wiedler * set sample_rate to default value of 0 in docs Signed-off-by: Igor Wiedler --- CHANGELOG.md | 1 + docs/tracing.md | 1 + pkg/tracing/elasticapm/elastic_apm.go | 8 +++++--- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dac014e34d..81388ef318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1939](https://github.com/thanos-io/thanos/pull/1939) Ruler: Add TLS and authentication support for query endpoints with the `--query.config` and `--query.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. - [#1982](https://github.com/thanos-io/thanos/pull/1982) Ruler: Add support for Alertmanager v2 API endpoints. - #2030 Query: Add `thanos_proxy_store_empty_stream_responses_total` metric for number of empty responses from stores. +- [#2049](https://github.com/thanos-io/thanos/pull/2049) Tracing: Support sampling on Elastic APM with new sample_rate setting. ### Changed diff --git a/docs/tracing.md b/docs/tracing.md index 8ac38948ce..84b2e19add 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -100,6 +100,7 @@ config: service_name: "" service_version: "" service_environment: "" + sample_rate: 0 ``` ### Lightstep diff --git a/pkg/tracing/elasticapm/elastic_apm.go b/pkg/tracing/elasticapm/elastic_apm.go index ee618f81be..0452ca1e7b 100644 --- a/pkg/tracing/elasticapm/elastic_apm.go +++ b/pkg/tracing/elasticapm/elastic_apm.go @@ -10,9 +10,10 @@ import ( ) type Config struct { - ServiceName string `yaml:"service_name"` - ServiceVersion string `yaml:"service_version"` - ServiceEnvironment string `yaml:"service_environment"` + ServiceName string `yaml:"service_name"` + ServiceVersion string `yaml:"service_version"` + ServiceEnvironment string `yaml:"service_environment"` + SampleRate float64 `yaml:"sample_rate"` } func NewTracer(conf []byte) (opentracing.Tracer, io.Closer, error) { @@ -28,6 +29,7 @@ func NewTracer(conf []byte) (opentracing.Tracer, io.Closer, error) { if err != nil { return nil, nil, err } + tracer.SetSampler(apm.NewRatioSampler(config.SampleRate)) return apmot.New(apmot.WithTracer(tracer)), tracerCloser{tracer}, nil } From b2f39ad6a4b26c90e2471a5948ee636b25ace644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ndor=20Guba?= Date: Tue, 28 Jan 2020 08:34:30 +0100 Subject: [PATCH 202/257] add Banzai Cloud to adopters (#2054) Signed-off-by: tarokkk --- website/data/adopters.yml | 3 +++ website/static/logos/banzaicloud.png | Bin 0 -> 45518 bytes 2 files changed, 3 insertions(+) create mode 100644 website/static/logos/banzaicloud.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index ce0e17cce6..386c1f7ea4 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -75,3 +75,6 @@ adopters: - name: World Wide Technology url: https://www.wwt.com logo: wwt.png +- name: Banzai Cloud + url: https://banzaicloud.com/ + logo: banzaicloud.png diff --git a/website/static/logos/banzaicloud.png b/website/static/logos/banzaicloud.png new file mode 100644 index 0000000000000000000000000000000000000000..4f10972ea2edee901e8ffda67312f75bfce52b0d GIT binary patch literal 45518 zcmeEt^;cV6&~9)m#fudPF2SKdDemqBcP~zHcXtXDmqKxOx8N=APH}g)o4()uez^a` z-D@Q|D{EzE&df72&z_kRp`;**j!J|I007XXrNmSK09Z``0EP(}5&F%H5h(}s1I0l~ z>l*-ohW+maGqvjb3;+ND(qh7D?wQ9vD4BZV$#*SU!-z20DZy072Xta7RKZl?`TNKt zhk?1V(eI*rElnjQ1$Qz6e&7Eoy^>eM#C#CLbrh$U#9O z>(FT$1K)MbzwlbWBwu^ESbugYwaa?@;$pWtL;wf#zbjd^D|giY|E>`V%<})eMy34! z*Z*|!|LDO)NXfFhGuH2ZjlU0O1DTZ-74V+l<4_VK>Mf8hjL6>zz)JPX0!siN@`gy{ zj~so{9Pzep2H?#-Hy{So1B5{SRsgsg5dOF|E8t7+`T8x%Y+2Lz%lPLORu9;Ji&JcX zfci7g@T~8*Y*k#u6n$9CXU73Z30W4bVJTpP%47qR!iK#_{!($98B+moZ`xe2h4$T} z=vgu|YjuymhwW8cl>TzN(}m7Jw@!uzvv$+LsO?5y}Wq=yK#>qVGsALr0E$4YKfD{6_s?N)M36aeQR^pK$0$?wIO)_YMTRa@kuu2^x9M3c!Bvx)r zw=I*vWNr4csIGH4-XR)v%2da*!UyUQde1F0n9%<4sKEj;QSB( zlpI5mawZVeTt27SFo?`hvgv|3;Pv@1$M+fu;QlvR@THjTclSSWUNHVi;~JgAX?xx* zJuLve%cH9BVfByxV<=HFUXu(7B+K^)Yr~VL zkm_8nN~!N`^c(I2G;;1QWXPTWXPXe{Hjf|cnAfF!@aEE(ng3YGBm zjvg)ZG8yyK0wi=p>T!*5E$!Mwv3R2DaNSoK8cdyTlR0M({k!T>I^vT50pr#aJ`iuh z;)ly3I@EHWeYPRCLP>%%jgk;l_HYy?aQ1vDa*a&J37j#45>L47b<&7B1|q5E^)|nt z`^~tZJ`+14Jaip5Aj5-l^NV?psua?APO1d>9hVGzu)F1IT{L=45ZcK7vwS(jqv>mF zZv&DT?8&HV7$~g0a}N@^bDkc2bmm!J77w|z?f(Axw;BAg-oM=_{y{)^@9n1P?d80N z_Ej%WAVlh95aw6}F=tTTlu;f$Z;zA1`nJ`7b^5ZFPkXk=$bg0nT?-&gWUx@x#iIS| z^`)cwbSDyE&kbs{6dM-lmezqKL z_u6Ajce8IO<^w{PVcfsqMN0WzKgVMc{mu5Bce4@_rM#ADVG%?oDvKeg{CE>^TWflQ zS#~W$@m6{GN%r;Ig{t5~DpOf%Fh|2bE#G28m8`_|&-Mk}a*{?9%l zVbuF$=CX3+mkc}=pd}>V*=+ZPV z1T60qsRI`fYE+hJeUz@Vs9XXA-mdOD3_9wdlG%zZ{1+z#gn^nc`oG(cLlwu`uOHi> zQA|XJ(TZ-;1R)w%%xAv_ww6Rw?4MUA7jDM+;8;+PqNddgRJV&%RaWT`eVu>3>m7XE z>H5mejK-4u9*UcN6R4ShO6S`A4hG&3JUV?~ayc$JxI{9PHBR5Nyd5G#$ttH$1LyD> zXm+G(ypxbKs@^&nx!rFSF=%^{bDH5}bh9+y7#aH?h}GUf^^OS*94GOh&Wr9dz{U%? zKS@#_#r6oTHOh7=Cv<}z(ei?4u{qn>BmJ&j#XFx9w|vq6jE48`=IEh-ah<_ici$KX zus-->7_!V-CzD&#Lu8T+hO}${c_wUr*Znv~PP5_sd3#u&@eKc}m$JYugp?CLekj1BYPlJa+3HyH zJXw4ayn3v=tNKsV{*mNR+uui*DQm3XJf*^HxM!97+a2S@$@VwF342!5M&Jo)ZI{MH zBtI2niWd4zBEOMjm$!YL>v(NaZY%RXX)Rg24aDnSkcU({(}4 zufI_=wB?IU#cL_uT(~oP^)Kd^4Hs3M=pxihY9Aw#DFjnxkAm{;msl3(6WhTws&i*= z4@CxDCFZL~St2MfUhnpFzdcC-ieddDaKvAo5p$;Wzkd7{Hvs)6dUkFdHXJv-Vw`HftvGM%Qq#Ba%Q(B%aFVtpJv| zvh+v4iVq_<+7Tz+s5qd7AmL-LtLXq0UqxMl)%V{oM@I~vPu`yY3=%h1EW^OGk>HH| znDu)mXDMFID%L8!UbU|@VAFWB;UeRk^|i^lovp9$*( zFkv!D`j@>?js7YnilygRdUR+d#)}8WpTPU?%`H@3+;0lx?fD}A5RY^~RP%@*4Sv#a!8Zl{)JK~5sY z{{C=K4<+$${u}#r#$*1szD;+WdT~M;r5GY}^&te{y4blNB++Xr8u{FDqv>(|mCpxo zRY&XRI{oR>(Yj@rKdI640Y}()^W)s;V3xp@O!x*LfB+y+``y=JrpQ=q6ekCvRJLbVrxjX9;W_>Qo~5D+=TGE8e3_2`hF9(0VvLuf7`gG^Eer z>bl%VCC6U>GOY*oCi&4*2rDtj*0Ya}P$kU$yf43OLcbzq0^Dyw4dLhZy3+Omr3adx zIsTbrWA`cb4(Vf(s3^u6Hlf3LjDOwX| zHj)7@RsGjzr&%*K7k znv%mb9UnSsQ;r)h_h-o<%ul(|+{?Qs-d7Jug3r6XNI6@J@2R{m*U8%Uq2l@?0Thrz z3@i&oAu48=OtX~YoyTKE=w^HE%b|rHb;giE|L2W0Yv%h?D3*GSe!)DEJcf|3zByjb z#I5`uPM!=$q2Z<(9*nM#o&xM}q!2h@D9b2dwun4fVyW`B$e=DA=5x+51nHU$9XjU? z@0!wXV>YlLCs73Dp%?lz?8s7RM7-0gMeU$5pkSwIeaF@H%es_~!2K|KoZor-{*D9= z3Ha;nhKZw-%kQYS<}BkvN47F}tw(U)DHB-t#71K0U3JkCIlwTui?zM1G(u!+cX{?p zi7d6h;=L0yZ2SYmI0HxESfT$5aZgY~jtSHm6nUXa zh9JSbpWzo6ve@CMwWM>MKh_P)AdA7fh;}^6-YdBg?f7pfS|LG}&q+ zJULNgx=HCL4gSgYV^j9q;)A`w+XhU{pKntw_%W>;%Yv`I&#cT2l8V3P2Y%sQmNJv<1<)5!4Ff&=n`T8~J8=IVKu!cXw&b0_lR^X_1d+dYNLb-9YF#MW~7U!runnq9Usp8k9}9O$$k@V7E(ei-;? zeOs@kR%!&-#w3EMckDe$wU?Hr5x&)_qgR$%Eg)=Uj7$x)_@Z}lAKoHiH}}@P{vhP& zwOI;RU@`y=$)McoiC)0x$R~&A*Vn_d>cIjvr$QXS!75#xX5OtV71%#qm_cyQt#Bw^ zy7Cz!Q;;lO-u$euB}=10+EJ<#pn^zV22M+sbxs_KA8?+GisO43vdvj;z4)V`>`vBK z&$8k@Ff+dXoOI6!n%{(FM5*mT@X{B z=6Wil!Q&(!A*cN=9Zerw1OfgS?w=YV&7~zl0xqLIvRl;&kvO2-?MMPwVPs1)>V&dY ziSo>6hML4c`jF*Cc70K~mq1hDv(*4VuWH0bf%Tyb|<9PZh9Vgkm(DaPO@BzOPRN;lpu zd8iaZlNb4y-i#1bRITqap8^{(5PLIRqCPyI<>-FvdBcW9gg@^8r1R$#>4TXDE^quGK>N>sBMtg z@szOVl$u^M;@+-D_e1`AaSL4jAqi)v%5(Wd=Wb%R9UW?Xl zu7i!6jkBQ6%a_rjmpDuspdpJm4-!KB6GUGxA)ms{Wi8kaU9dUZ$R3yeg941!g37g)Ia!f=)C4Zod>x7$Xkpc0Z=+fWlYL~n9+E+O;1~YQ?x^(>!e9l7Jkp8&T*;WE`MFPj z#(IYwT#SJfL+W)%HukigS!cx3|6`Z~QU4B-@G}Q z8@p#3EatAC+)}lJB{m9*%3a_U#^JWu%vDha(+od}b!aSLyxaH|b)*Kg0i zhhC)XwBcStLJJftapxsEewP)Fw2B=Z>y#KZk&~-gL1SZf`>}6F>=TQ#=&P%%zoka+ z6JKARgx`FXH_!ZESq+8!_62jvsEI3UddVvF%`q8jNrt8#Zc{XvCEknQl-9&wdYRo2 z;WC$X`7~hU-U+XaM(Sz{RI*f+)5&kcRVY5N$S-qsxV%ostzDoVixY5o4E#`hzN#XH z@lR{qqC=&7(Xfd#vn7UIMhD$kiB5*w%r}Mn>eoEMTMqRhhVQaI$qylGIfKRkOc7`$ zMEoDpdxlj7JXZ#*p4NyF4aM9v-Y+e;<^KGM#`~r9%5UpTFG~;8faCK|On@;{gwe)4 zCeA2JCqnNm1|m8U01oNBKMdScU#{?r(MB%JgrDJCFYF&a{q!c(+M&q7iP1PzdipT6 z0Eu$G9iQ5-5C%~&xhNtuMCWQ1$Uzxz%Dp}+KTqnocp&U%#EGA<)` z@3cHgk-N&EnXT$|tCG>?*cmw$eUHOFzuWcl_X&R$(!Zw1b5)w5z`Hdc4IT5@QbeCG|LqKi0 z#0I6r#&iGygb*3Ev;QHrnsXG8$Weyrk8H{jcxR$j?O;+xH7w&oj|xr-(2`+x?b086 zpSv^zzpF1I)dB-)8mpWS(I9(oE?48l6nGeDtQL~uFm2aB?R)Ms5FtpLkZ#A8BwfZrD@L97_SQN20 zJVDe5LN1!;_x4JG;m`t+4AES?Z7C;L@K+2mrWdt=xD&3! zKJ!smb@hDB=(o6)}Hylk#c=cIYvIZav$b(4xg zQ$C=SMlp^ioA;$Rsw(|u7I~x39U*e?%x%0fG=h|at z&Qp+3p?0a;=6Fvm0RLXVhUXojFJ>}YGgg?Kap5QBF;^h70T(pEiBschfesp`PIm;} z8!GNa^%OA5+J|`ma=n^j;lIkTU!#SuWF2e194O_eY`=&DIPg(vPayke)1=8Qt}z-G z0C-yR?sWUN>TY+1?&b2SzZC7sf_(N~yE+^Wp?upl5?T$sg_hZuqS#z}hhHimFSBy0 z()nH*kqr~$X8<$Y4ZS!lO`G*%8OFHo z?cw;-@j)-z>Eq64$sA8tZ})wpm8X0xrWj2Bzsbh9G%p%+w9>Gwfq`9ZUK)4iRxV%J zQ|0R;8#d_&BT{z%sEA>$`;f9Lg+s5MUc!A}yA%ZPWyqmLaZ(Xz_GIaYqK@>g>uqnC zmR3I7w<}J!KT5_{4{hN@h^1xZ^JTKaQ#s_hLk~kn8CosA_bF7A zYl`cFX*!6e;GIYZDiV9Gp61#?25jovWH}rpVs476kbElYt%$o#u1i7up^e9;Bn$6 zH`z(J>p1o`2+L82IqVUa8iVUTqSG>+Y9UA_gpn~|KSGh)MBu6C==9zn6IyDJ}E^4f1Oy)T}neu~}+Wr3L@CHv8@j+$ms_y1GK6wnLIQ zpg@T&3Fj@O?bj!Do)F?W9uhTN2q!mIRv4xC!9r`c3IBdK%yNCWY`k^D0OhmR`epq0 zp2XRrM?s&~>IGE<R-cNFWda?y-#)v^OyPi2VV~oSV*9Lc#(h!=e>r%NsncR z*FT-Vvz(+z94TSiR?02w9(bE?+n0^l(^FjbO3^F>P)s zqjpH7f~?PyQi;qjukaM0R(CiU-W*!V+FSKI2^o(>gqGkrpnlg!@MUxRRn=VYyaXCN zd`zB2{A!(9u|?Z)0M&)Aw0w2$lp|FnRU5M0zbW$DVYMKU#F{_n)!A!A;4za@x0g1^ zgABL#MGg1E^LWVt5^8)98Qd8uJiGKAmR;z{{d}l<=Vk~2wPs4XqCBWb9JS=~2BxVj z28UL?)7RQ(KwAC_m%#O<>iUz@R%{GEs*1o%WEGju@o%&)L8s^8)Md^Xg73SNi5FTj z>eAYwHD@J~QGWRH$6o@@5%o0wRX=tyf>Tk3o6-%|UiOc+UdgK3KltOC3KrP?vB6zW2pN0;36y8onAQP8|ipa2bzB zR`VARqDV|Cu**uo>HsJ)z!sMIs`kLYjXFpYQO_?`t_$wRfzm#ToH@nGc~Ycm{1a}4 z$d$X)x63E{oF?1RsSlHz?=oHc=Ld73Zw3*7Sh;V>(x?DVcFKqpm1G-A_}(I`=5l&U zO#OmnGUT_Vmw6xW!&Y&$R`Gsl*%XC*$OmH+nyeJtwo4bIe6~kg2GJjnr?&1;=4L+O zqZ`J(E9C_FlX!_7i+}RhLs|JWfbjDJrN(W4f67y#m@uV=L_*U!`FsEMb|*AWJA1hx zCs`E3yN-A*bVEs>?%Ky8bza2CyU=1D=q6El@aP$l2-P4_wHAP@eHT*+CrVA^K3do~ z@7e<{H~{D*y8o*|Y`;drOk>E(pvoV2)F(s<$b z{;ap2WI#6mAFL?GW;fvSbwUVg@qA=bIvUj1&R&(A^?Q_yGFQ)jx(Rs4u|3pnX2y2F zizhOQpkgL$nvjot4fe-jPqw6PDgxp}eD6#P_w4rT5W)N8L^)EHSic%_8H*+-jutBEL_>)#zB?5{G z@Ag)*YCD*SUU8Xj&Yp_zU+hgz;f{^>J$NDOx@yzaBS2}1#9(+Kw$*Jf5HcdvbPFzK z5Q`EmiS?{I<^d2R^=D8c^N_QnPS*0ua~5G^hzkj8fA1vh%mii`FSgsVW)bygn2km@ znz^(^KoH)h)~p=~t-kjW(Mx-siJyO-=dtht{q_e_Dz5 zPmf2az`fQ8GGv-?>4TE-qGUA;yAAAGp#y;LPtLg_<(gPx6eKka^VaI+>kB|h{VF_i zu+?Mn&$X+QJA->Ym#DpB)nAfi5&RkegZRfp6kekQur4Kz8rUZ@*v_^JK8jq8|E=o$ z1B_G}n+NdQT1;IV%gBBekj;~n#pr`Qxy0K00q%#p1r7_J8@Ty3 zHW@Ws%xYrHz!f)~e}{upsoLoblfV~UO4a=kqSSZ zMnrac=@0tF2xOG}5zREE!^P8ELB z@saJp++nq4U0e=;zul{+)L1Z~_VXaV>ldwDsajGATtIIf+L8D5OFR(*JRRvi*W*8t zAhx!(8XbjJOiV8ITnDxGT+6mwW$IGn*SH~%#+M@R6m^vs4x)1E!Of-S=o;yz%2sxJ zqD8{j=)i*Qx=>nrvw@iyPg>9040dipvj;hkRg;ok+#O5;cV#Z?Ch$0d@PBPJtVN?!m$_!zIT=Bn>QZe#7QU`As(%nS=X>Q*j2)Kn%}p** z{pnsmY{y& z3Z60Ly6u#RNNZ)8L!7t5&AaKFO%t%`!Ll_vRw_373Ynue&J38*8N zfKO$?|L_y0M}+u{Nx2l>;cWt~JT>u@p=2pJ&74810XghNPLRaAfCR~psBlOaYLO4p z5mgGp_Y4l%ctV6oA%P0zY7YtCS`e9fo$~d}YZz*$K%S^Ex<^*ds#!^Is^QqYAOkYD zQ~IJC;t4X_V&O+=uN$;;FDY`qPFrT%sAyCAD^`VyRuzR{d>_%|5?eH_?_78-9WQt; z(|nB`QuI`&$|G>oRt4X`rJCg@ikZ3Aay}o%vSimjm#o`-K=Aax%k;aD0`Qnkg|P^* z!Sc`uaClx{sQ;A6QNr^Yd5`2DMCl(y9vFmXMDCoM8YNWyh-}cZ)g{R7G8cl8vRxx$lIv0K@LE~< z@wrQog+-=ZPDIUymztG}uptVCpkXavKZAzt$8ntztJ%I#1YEo3Ggbfv=ur_9s z7!2x9pnjYPkxtZXL`5@YXi!N=$wYh;bWZ+~mmQ6%u&jm9AH+Wmc%PnQk@ z*up)y@jxLJKc6R~Cy{!M31bpSer>VX9rM5jWc!+*>U|MQ^Y3^cV0Z%qJsFdEKCNO5 zGFpoK8bqzu2*d&bn(zX!zaHsqMBvXA%sZ0gSJ&bpM+RKky?L~QlaUQs(lqIvOg*%u zrXT#Q-c6qOVT2|~l7e|8Wy#SKEEOobnm6H8+;!V z`^)Ib(9Z7F29qCh^nvOA21*{upzVsvNcSCBhBzn~0C#VWtBNqc_k^GIg@O7Y#J*|0 zHMhCho>AGdSx+hberxK1rHSWGURiQ5GgF4dBA~WBFX_?FQiDfDmhhiv(U>c|$2OFNZihbvR?n zoNRdmF4C;}6+M5tadwo-_}m-hcNc*qo(GTf>-R^lqooX%42RulVwn8tV!2v=Jwz5Z zCwk*VQ?EMj0(9VrIN&PfkJm97S?3NKn)o7gI8LaVA$oa_?>VGYb=(s$Q313g%9)oT zCna!v1j&{e^|$g$3$WH0yI4u2XbHa5nJcB;M7yVlIT$+4x-AEAp?gf5!g;(SVkeM@ z6GVNa1~=0UL;KiL%swm7+5@6j#X=tum#u{kQoHJOe0E&$S@QjQDMk*Wqyk^4S~(%7 zu-N0HCem6c>#dAd8r2oOnC!JYh&Vy8oUA8_$<#O@|dTbCN6K54U$l~n+iXzH9C3jjyLYv`i_Tp*xX<>tV;~h^1(6=K( za+z*YXzK?LLJci)-6n#%Zau%Z!%D7D@n36iUKi=>BV}yi^IdsEX{6o8@$hc!CrsG9 z@U3_5%ElpP37?d!lFg&_1dD-*RJfGj2m7U~rSdVOuykNYC@LLiyH4XrM9&0D##k2>3;5C>!!ex)vyUV$%bk) zwS6#o;`a8u$U};KA0r&DUO?1s3>ndXym@xZEh(g=W|*_wG)^$Kyjug_I)&qnjq{Tg z^A7`*l)q=g5wIO%@@$8TUdiC}GhjmW8ERm^%%Rp$^m`IQ){<-|kQ+t^6?WQ<8DVUf zA0yN==hkM?L5;`$W0fDXo+DS_5HW)9^tfg~}~Sn=Zf$C!7@T zxNA*@7mwVCE*Th^_NUO<2c=ThHd6`LM_#AWK3pO{c$11KdStI*G}qcTBd3JDt7cT*#jH)0Ej2?4b*TyO1jDdMR^m+pyI&f#QArHNf^`HP zS+x#yNHD}>MMU_D%@JJvn^T94cDTF6<2SLb`U2UX3S!Dr*b8S-rt z562qLKW>;ba8>vy9LEz_te5+JS%{!&668QR+(|dW<$%w+3Q3~m;Ckj5MmB6{+c38P z==#Ngy#rjlp0(P-*Yfr3SR#qniSS-G0#(u>^ED01ct9$~u>~-PA>3#qln85Fnpx?> z;6&5T)${s1w`CgcSO^X6E+GDuZyYk`*7&6Ndm6|oeYssPL#UvCqV6tV;)DK+c@5)mgPE2lDs zCuD6;vJMVL?hAi&2@3Lm?gdY7OhaP69#h7#{eqYH!k^{KdfySi(89jBJwoT^0RckQvn+V7tVc^uP>lra( z!MKQZ79eHNkKMOre*DT;4+{Iqlbgtgd4mt=k6x-pWDXnAyOOdC)9pf5Nh zp`(gc?zfYrhX0JvO_SyGz23ZY=tm4aZA)lFg0EWR;^&r>M|d}uI@dzROEou6ld0@S zg&_Q<9CnO%ycPU1U#~mVw5~O@K0vI_30V?SYKI)>M@r7PP0hAinvHGw`MCU;+B+H@ z-x<)ZO!@CJX7d~Ko&Qceid9~(8v0h}k#-w2U(~R1Ep*gpc4;!d4$!@ul{9#mc2phv zEcYE_Ulkq$%<``q3wQR~<>yPW);B&c)(%FN`V;7I@f=!&kKH(T5;$m}`R9zXvG)8d zd)%f5d#xmq9~{CWagf9M*D!LZNTvuI(?y)7|3OCasF3tW?;g*&&-RZ=R)=$-Oz9MZ znNFd5`Mn#!wV00MbqT+nMAHU@30cVV1>I~F6yrmqZ1^+TLnDmukFT4Ga;4x|E+JVj z3kr7#2!hQEXUQUt^-=JOpLMr>D`H;Av$O!C$9MF)bV!y;_@O;45T%Do;)LTNxKkM&6J5P4c4S!T3+h_~){KiBno2OYOM455R;gHFhG zKuEs%`sy5aKsz7~p}~Qr1MZ|m+LJ4RiGEpNz zdWcQ{**N+33oZP`ywyH_&~?S17k8|WcwKa!`|*jx$#I&Kc2>)#KIZ#i5awveVZ`w4C6mj&_KAqvs`+YqpCt6$}}yCDEM$L z?*k>(w**;F0TCrrtJJtOkpaZy{A|u))8+(qY2mrKJRn#6m~@SqdU=-$Ujp6sQH4TG zz>f9d#*JH35r!;;e53LB^R#4NGi6xy?zLwP)qc_U?`e8hbF`>C0Zs;Y0tOazQgf2Vk1lY(9OIE()=r%U! zO8`|0GNumi7(P7`S+38fP|+{r4q%h1PJw|0o$?Z2PM1*`pP*y5c*Qms^g02`17IMD zqBz+wICppfmC7kEf3@XfyIuz+nb%I}gl#}wXafwblfAbK^$g>>kby3`-4SNlYO?D@ z_SYStMeEg0(|kjOSNwQ}8Z(u@PDWkh z`MpTB=a?WNZkDbOduRK{zoPtW5OD~}`!%=vX|WwYhm=wLLEu8^g+!<`z;R$2n>9E^ z_)ZJ76y_NgRPY#oqIJpl{8p(ThLb2TQnL{}zWr{-wm9@Q1x%eWjc?eNve`gECC7Nk zStSf%Ft+{scTP(D@U~Lq$B&*!_I7X~Cg`*w|4|9KkbN!VC<6`Vq%lsQ&t_tUf~44!_;!r$6ey z(CTi#5iu9B{eFY^;%`hOT)5NDDTZ~5O5Rq$>ph;-4}4wEMNu7VxwAj@@Q&*MJP8Q_ zHTfY0szqd%*{b8&6<8o(Rde1nDKVwD^o=$Qwbg~!;?(oK1SPyM^ESXs8ql^GMWV-6 z(Syi}hh~^6M*Xwh@!(e*@!_0?sD@+_5bQ(LtUH?pY^vbq zGJ#7Rb~7YQ$x^RG@f`&TQ>C}t_~m2$hs;;^ipJkHs^&gNWAkpWsdP9>&}oQSQC-iS z`A&F4iC=|DgvVFVQC^|MF%4EX^!X%N6x1v2!kvW}QM z@Q>0mADrdN)vA(J1jm4LdgedM=Vi9Hvcv0nX48(kax}48>Myte6(oA+b}l~I@2nsJ z@s7+nDRJ~oZ9macUQWd=*BAfR_emq&nPZL9SE$(NMd)s0C;X8_4R({H9~^}HZYx?? zT91Bmppu!J7|0knkV+4xC8V4)khfd&W5Ss4ppk87X4cn7A8)TEe`EKi2n1?}pk1si zuT8*wb}_$b5Y=xGi>kT{Y-a!Z^AB4W_o!sFw^$kNTKV%`SJ&0eud}FF#B1o2J2!yA zXu*>80@d-yC+}g?$|*xpZV!YSkt=O#a9}};b&H*iv+SL=V=cuLu1Tyl`-1fjB{-Mh zOAG{HJ6aIRG}s8YBl3N&Ol_wVI?6JWhbTWcZ04PspC z_t||kVUkhVptZ~=E4E4DmL2+=pfwg?v)a_)(xb{!9%5G3xHFFlHv`%+RR~XU%(AjP zmb#LHHj#dDO*vO?d|I|76%kI}Ucjf+7RlT0EULA@Vq%a^-wGY~J_{I+R6x8&{iivT z!5mHmKy1UeBy5oRk8|qp%#CmeHBM1Fra72$_(-*1hv&pG+c<+~v?FFa9FnxF)J^Sw zN1q7A$z8v9vvo91#zv~q-1rq=Zi-hl`g8K|{046@y{6nXGu@3-Xu6RkP_wXgu3hII z2m2ik2y?kHQn>FS(xU>m&%pUhe2EO>c{b4X-1DZ```+Z4bAEaDW@2p-AwQu2`_;r> z+jY*~e2CI$7$d%RzxROtjrPsyfsm4KfLBDagz7fsh#tPD+T5`i$T?W1#Vk!p?Yv%@57O(O+y6x@~Z-Czin0T}W6A;!PvaSl#&x zW?T`MYv+m3HFTWz6h#gkQ%wmT3n(y*-v(>oz?vf<%?t+%yQ5)B-1 z-+h;uXSBuTZn1_d8Z26Bo+ohZgl}YmK(rU4rdD?4DUIA7?(*M;uiW zQL5;Y?PQ9&nZ0^7az`C5gM#RSbrRUOAxrxLvhouTKZe+D54~RRkG6hgatI;gPEf*& z-vXx1^&LtUdS!F_wE}i)C?0l3^0AnclA;_ogGjsI5AhWBjFx^3{VtD>^m7lV3_*Xx zbjOn14NpSsV~J!kZeDxnjp9sRvf^}1`X6khuM0qV>++JI%2;Ne`|v(gi^XtXdc;pJ z;rROE=9E{CEJ3BruWLfdYUDAFKgW!8^$82cr`QLUhYc@#AZ6A^c{L5It0>3nf;RCu z5BZ!qHgOi*r*afD^Nmg1-SG*1hR<5+2ZUnYA1~K`4q*ik9dZogFTVcFQM(n0`LQ<3 zmH4P)&stCgBXdPgW4og^I5%g{BH_+R>5YqPJ0Dx#F@EteHj^}Zwu4&6%EwB$2tNbm z05*wH$DY?K@+NiIM#(tyXF~^?zJW5jHP>7bQKSA+U@n%iJK`VHhw{6z(A##6*=)bi z@!ndhffRU^sl!1!vrJM)`ZCpW#dP8-4EI>Psi%axI?K;){cAeKp=ttm)8VhVZjVgA zL017~XAgb$u{=goc$>4l9g^pU$q}QK&r89`ztz0!#P|j5^Mj*m`@(HI;brS4+{v#iwuo;;n%@7HdSy7Sz(c$~7kQ7lR&HhM94k@w1 zE#O?fZD})Zyge%(MDvr$FsGhNn*e5&H+p>N>(^5`1jnlcS9v>P5p%*7zVLfSrrkgE zF*kG}_RI0RyFZI6>J~4whTB`uLnRG7rJNaZZQujdX=t*X=8DeaIBY?feA^KzdGi!; z^CLuDQocKdh|sw;fK0l1w-T{=6MXfw3BH=K*^zo_uS5e~4R)|1E%=9SA=8h*oGj0P z39w<}+}sK*`t0%o#Sw$sg?(X;1p*zJIxI*2@8(}R0TQt~EilK|;Ru!XRzeLAS%-{P z)@Jhj_0G---m%R*FILk5{#Z2Xp#|)14bp=v-EE4@WA=ngeCY{TIxaq#$Z={A(+^GY ztflowHm9{Mlq0Bpv5(PDgW)L`Hs2nQ)Ax~D*8L@nm;6`fRFY1I93(%-Ch+^eaDAodT z=^5NYO!KgERvMbI8QcPlM@6Qv(3+40E(9@vZDMHx{)_dInlUxpLd&e!KwWB4HQcRs zUJXzk>$4QsO~`Kufdk9?Y8Dwk#yJ~nf9OCJ{!`!HhcbC;z*h>pA1|~{&5KD#BR!Gp z9ms8s+`K*vVxzZaekj5CeU-&(_0TSM9IjL#P51{p?uX;>6p5i?#;mMW*0+aQC5`n! zjuNq`ed-`ft;qSMCGn9jy#*^Qhmr6ygrC7)vcUEqKswCqwKc?8{%^ZoKIdnr1hn5V zmDpi>pbu#1Jv8)D`dg*X%h)_nlO6bS>xH*NPzN=D*HJ>$pqZ;YOi+DbA%r2n>zBJg z4eFEBJA?1tM8{6B3m)pBGdfeQK-RQK#A-_8}`xvlte`s$^DUel$b3ftgqmj_M`%Ey2kCwaFFK26-l9ui$k!D+|K39dBr;U7i~bSdq9Fd$d)|7G{6ipve$qO5^z6)_~bhg z4m#eKN*{{dBpRe%ZzWYfwVxxVTWCIbHNo#z?yE%FT6NgMIo&H+@Y+ z%Lo1hBt>0|dkO~6MNVQUhy9Zy+vxoqT^|1w?q>ai2H161Lj~w~7vtQ>hKvIPyD^koOf;7)5h`>N zfR+O?H3NXHzq?ZiA(ezGL+9GVVHD3>wotc=oYiSx=+fz+d|Q*7VFfAjOEgRWjhQe7 z6e;3co9(37qEH<$xqLBtu+IrQt+bONooLLX8;LGI_+#zn{TGY2L?os9D~}eeBC#BP zC-f18D3SW9Qe8!#qMobGX|qX#w&|rg6h#(qVj`jwWz^4}e%43P^zA+{OsFN%C7nOG z%j!77E-w|;4A(tPvGb`_V{|d32)Q+?JEYDWl|`<@8);B4RdP@Ha2!P8{4++-@HFh zeXE{7V5X|OUVZwWd(S!T{o+k%VtG$r_k$vH4MnXJnD!*58|KP8b-a{NulzB0mDWi+ zj%x#r?a6}{`Y7?S18bJ(reE>nUqfHWUxAYL-uuGVhNh?Plav-2DL;SAUi6>$-fv-dVZr6|MOnP&?b${FoyKxs**p%}9D z<&jf5AqQ6~66fDo8r_Qf58Rbi-mUU52T?y&!VasLE2VLtG4F(bOo_b&>>u8lX)i;*ayXoOfj8GYqZvan$*3-kY3pI_3T0z>lYnudc zyIdxQIA02KkXYv^;z_MCd`@{0%Y|qXI(2+krjeRSzXntL-6i@4m}nGOQ;ODineb;c z?n#EANfYQ5C#2>WIhmPz#O7FZQJAPO=zwn`%+x~+@07@-iY3wTANpg-L7m9GbebGh zDCM)RW{;-3U(Nzi5ZD=A#47d?3uK4)=kO}`OkC`U_uAWyo{suwnY1iI`FMufTb%I) z?6-zgB8IE9wn=$qDHa6elkFk|bGUzd=VD;A*!xhbakja@hovW+@;+l72iJaV8_?&3 z$kaJa1jLBxpr|FCa4z=bZNv6HHEgQ3hTnyt!my7fcTxzTu92sf)6y98Y{8~d#LI`` zwHp0QDWI|Jq7vq{Wpn=8=5P381chsYbmVt)5Rh$+89%;=G?GAmhK~Oxl(Zsptb{gV zrC$OuH`Bej&p_W7*S*OV>tsB=xKz6MeI)(VT~PWwzkJBwZEUiIQ2ar^wxVU7D$L>b28V*dZr2Auj=jteTkgy#x>s}ky zC^k%RAMvK*@_bYylxaK1~zIaH*5UnLZ)yZsAAu6j*3NmneU zg*_ahO=gvS0n&VHJ%d1y621LpkMsBa};`u`8uXM#C@6r-p>F4AyiV zJKpagti=Cg}FT8;`m+2hz+)P9In{8DDlu0fQu?v_O4GYj-;lnYlt9y?DI#@AS( zJqD2}+KW7J1nLz57Q>Oi4v#Y*;xjuZ2$)(6 zj7kKU+Fd4r^lUg={PD<$02e=gisbiimOrX8TA-S1*0O9zzlH2B^XyH*X*)n}hs~KDs*%e+H6NJ5 z-*I?~c2$X1bq8A(tGDGu*)g*XU8|NN8hCSF_F{-26oi$-0>h7u$U1$$H%??@x(w0f zmi2VGP235SbxxH}wSyGLBqU{7l0R6l_zw-<(5KrVSita4O#9jl2|({bSu%lkW^Xv| z|A`{PNlgos$x3~LuTI8Wp!Kax!|t)Cs;h8?=$OT3M_W0y%S-=qR_s|NW&m-M1L9(q z#fVwQ?D-d|uAM$h_56#us|=aCeRc;_+a>gt zAS@5?+j@*_7|wFS>FT~6#?5yUWxB!m#*gmP;q!G{c+WZAOL#ea4!&IRKa@9s?He{d zEY{EvmapP{Q}3R1)aow-uh`Jk3gd$N(Hwr#J2*?ThK@W@yXiD*j<{@tkxCVYdv>S6 z_5W~Poe$0b?iy2&3JU28JeS3|8lCiaGllvVr5!Q`Q~TNp+#`U9Arfdc{Pyy8&%uvm zh-Ah zCymh*x)#xxdMFHEq;6{l!hJXl;q$`wG|+AJDDvp5rX&ofJOL2bm5;f+rN5S2@vNej zt)8L8#NrG^6S_nZ`Y^2O(r^Ly@;x$!av6&|W*YH_n#V-sV zt7zk;o(*DhsRd>{s6`KrUZHLUda3`z<);51F3-Jk@S^$8U8|E~5hMJ!>KR8qJttRV zXz@a@*JPd;UMtqCsH?&#`vH1;Q0gSkcWEvXq+Gn$CPde?l>15 zBykx7T)#9@)yyd)gCuOhoz#%K$POBNt-)5rrYlnP8w~ogi7&#zPy9R7+Vxosg$v{f0}XjcXfPyCxq_N#R03~Rzvn~t5g1wFZqa}KsrBl=|~5;qxXW7e;sy5 zTCB4_?)+N>xr#ZXABGf3ngpB*Qg=dKYC6uvrhh|VgKz!t@6WmeHsZ84C5tjzVKQI> zX7jy{$8})K^cr095Y){`ef<#OBi--ZfSCux>GkOR&?X)$_+9Ng({n|pD6|@6s>}o~ zl;#X2vQn?Kzduv-`8IJR^iJ&OwYd{|CGi-FjvW zKA=|!){+y;H&VExF!}B>Q`jjE5IpebFTElg7bxX%JV5%Qf z=xGzqS+&3=JNIQM1JcgDPsC+UOlfbD*@n&I+XLARVbF8;I;y&}N+I<~ zxSIz%fe*Vw#PUkBON==vcie;&IQ<%y28Coa@9_iyW?f}|F2s!zqYvt~PLVI=PL1b> z2IaRS4O1IyGTQg{gayw#n;b>qf9Y1kd)(qz2cg-U=<1&$(#+M4@2l;9)kHv+1d;ib479+(TolvjOc~HoZ{}AKnoKS4DhPaYA$cqPN*R7)x=vuTGF9%T zR0msU=ML>f#*QZ_yF=x-47@SyUKKq{gx&-6S{A*pKMKMbZ0{I zOellnZayxgz$vrV1ere*jMIg?EMWo7ld?Fuh zqQ(jM;=NLbpX^JBQUgbU>D%#e008w3QpDjzV!pbAzkIbCnvAaQ;&dHi^S>-!QUpSr zPHf9idt_@_3;>bM9v)nMmOu>|sX>TTFj!WVNUmnFd%!d`GbM!7A-u1bE&T^9ioYjR z?j~{|t*duFR9SC-K?a~oKO?Ua8p@xeEL>*!U;rj^U10tyy<4hMGG+R^{t%OsEYIik zJxlqtv{?5Q=W)4(jm=fju+hWpyxvNls;~OuMO)>RO2WjqUsv5G3CP|YI$oQXywuvB zrWjb^;;*zb)Z2Y#IE=iKsq;bEr1(|kR6xA~9oTf*+~kq`zoiAHII)zUV#F3^?4@O}FSLzr|sO*~qiN`n-TU=hXj19Y`k<3KnNHO*0yz+cZnQ20R9)1Z2`<}n zV=QrZ&J~7C^J424nY|^gKiJwQC&@cmcM#8JHC`YYp0IUM5L`K9J@&5$mh_O+`PJK+ z-s{u8+Hk+;D_`Xmk>i;7d}bA`CSNtCe+lDlw1?&97`uvg@=UcmbzxKU8`5UtUIS@~ zB8_bEX~cjoH>n{ld;ss{2&QQaoRj($lpV8l;n)Sc=<1mFjF<2AKXFna#IcDn2E+uGQZcYh#T&690UX`-1(4@V5q5a_|6tUCD^eCx14vXSAuSz@;#YrwdRjQj34b-(a5uPVCXMCWK|nQQGf&=|!QrLVZY1LT z>fg&z^GFjOs+F@`d-jNCf^(F6dqn9c{9ws*SthKmJ{zhyP93$lbxJ##7^u(?GIlg3-SBZRDphyusUDT5USL-@I0Szs1Hi zhDJ4dg`N`s%f-q%a)khkCX4gCC+5L*wz~jkP3nvU(NDLz{QkN51aE=HThNk7ATD)( zWa>Ce^MYrt=Y9Coku--JuhO;ly%vN|1>?&Lv0aQKfj-W)d$t4!sd`w|V7G*6YJQrz z6UdKZC`rc|m`(WmDdi!Sm!P=4&Z|e=YXpe|afOb9354OS&Rv$Ww@ZRr&08Le@A6B*r{s07 zkI(@WK9SixEBrkyc`i&SbZX@zF^J_ZZ5(7rSmB}z(-)8TJ%Jyz9T9^HcHwHaxUbrH zq}<2Noy`0fCgPZZ#vBOte$PxzIEWw$gQnazo1B{={qAFk`GXa_MIGQznvyuBwA1Bj z1&d=f85=Xxd=Parx`^Ryd8hCmRSp^vY~T_@{$0tmnn`+@W~DYyF?=~IKu#~Qhh-QT zMhCDdraT+hWM0@?uC>BDXX}AeEN0@vn~V-4AG`?*Pl7+aVJ&T~07O({;b~aSyXDKs za{;f3P#=)9+4qiLXiQ?tFeu0GkM(_w=eVS8eUIJ;?Aph$z98!UTCJnCSuP9+vX`p| zF@nk-R*1jPznnDjf`os|ria}@DiS6xZw`&6W*Y8SOj!P#`u=xQBiORe_e_$KMoZ*C zMIl5|9*N`!8Vpj5k|EdoHRB&~5YPD{Y<3$Ig#_xUNDSh9G7-DHfUZOmt7xZI@qmT$ zb88A%jrm;~4jixu2ZWD+XqTl{eB#+U<;uPeP0_6DOqd>eqC{0VbwpJ9=W_NHl2(EW zZw{ZQW-%&V@s&~@iU#;}EOeFCr?8^_A|H{8F>tXmx0kHXPEIG=fGDd59>gy8@5b;s z*^HKNgIyD`qUEq*JotN^j6e@7D~~ zrzvz@X%#{?o{E>W6L2A-j?CR5H|f$Ejkf}2Jf#F*bWBI=ji)@~p~Fm)-?gb(+9p!K z8V0#_F^VbVtx$$0X%}y!Y6IrJLpu?6HZGTnQY3!KkYM`j*f=sNomn{u^c950!`?G5 z-mIJKEJ8tf$YS-SiDis}Z7}FE{_iB|HW^a#&O|VyLt6fRk*f-)1f}*EA6j^FqDDT! z25N%n_}&4@`$VR?wl9zqm*ZW)d2b78ClS;^ zQcx_ML<;heA6E$&j7&Sv@K;X~s?e!HGL;8=00ffPMgeXTHu z6K83>rZ|~7Gjj7B61&ZA%B}=?T;Up@r||b<;)t7@et{;QU$FfP$9_C$-A|n5tlV^L z*!>t(X3=O>AA#^m!ky$bFkm_qrS_}X!G)JSHn}rw99|1g@7;~HTf`^IYbCy!6`UzFoFEH9}8{>>dhm)M!5iT9cXcECGOMt{f zQzkIEV)!EhhpGPnDkgZ86x;UzDW~%2WK3{Cx4;6El6lF7^$uns%hpm zI7I9OheXJyWN^d2V|2x2f7dCKTb4M-4@?-+;^wu=MT&8Z9LQ8l(air?;X|j$oF@D0FQj9&r;6@Y?-e3&ZYj zb(?u=ZD?V`OUaW!0GK+UILTdIDSoHXi7B$0C}XfPjEHM8qV|?>PHv07&&L+7N4QNN zgDP^8PicP1e*OiD(_<>N-;5k|Sa>VmOpa+g)4bxX zd6mwYS5>^ec${MSe1Xcd;rL@)0l3c}r2 zn7nU`+K*;JG%4b~kR z&&)6(=;xPrYaB6Tbz994nBy?0Vwdekma(}47noSWO3_;+x@1wjKXD~EnlDqcQyZ<( zchZh1gEq#YlH=vWwazPYkI;R)+_VQV6QyWO+=v5nlL1y%Rw1|so&eH*NmoGd495Cb z??DV@imvX!(X!Z|RN8Zx=cu8AJ0l@jJBiak?ZS@S@1sd#Fww`)U!*Pg>?B9F+H#en z+H+XjtSXpjC09%_mj0mf~L*DLvv4NHr9^L7b~KqeBgj&?hlvR>q@ zRcY&L`o`3e(Gw@gh8E#CNc`?WkC}uXkUVu5P~jt`bdn&@T5T$NPcr`-kG^_&cK9}99?Y;h=Y*i`BDhno z=aB%kBGVN7AB}6{U3Je!hrbsc(ZPvTl*EqP^7(%1-n3@M@&h@b77}j+ZI3f5jo5fg z`uSv144?%li{rFSAKpONIGVettRGDF(r-LZCsL>I1*Oa38Fp9|M{McwC*{<(zdJFQ>-;T?ed^mRLy8mwVa2^1yMq~1z2cr&zts%4$f zs#vVRtGk)+Us1X!X{_bT;vpxS)a1i+-Y)KBQ7PpuN}ka3$EAPJ9anr&gTZN8sAGLF z<%gbN=S9;Jd0uKw-Jkf;m?hCT^Y_Ta0IJVH(*QD)!(JXv8KpeTXGR_V{aWh*kzCPM z!TRfR=Rjr{fVqgsYfh=aj&yqFu$xlVu?MCvB!eaJzpm;v2f27ptj@OL^tq6`yk|-6 zVpL0hLZdP)Y7n|8EPAHi{(%4gX#pm1F|XFSe(M7$>%Y38^ZD{isYzVnoj<4M)2+y)gS28@^qe zt!K%0l^zwn6Y1Dko`D(n7Z40c37HK#?RLCKk#6+lR%kq1+qu>P!@NrZy?|XhdxXpU z!a{G7MmqIkQrKTtfEa+Dj7LrzV6>X>FF+dD;}&r!dh0$?8}!X}1cDs8Y@pxWfY?t2 z=t)#X^`q=SZZd_@B$QJ&O?5W44N_s(I-p@3;@siN&6rnVKYgZZ?vLx^>D2nD`yl_$r^HPtS~2Zb=ML#ANhnx z=H;zH=B@5k!bjMEovcGNjZH&rJ`MSFwti97DmuyLaSkrxb;CPbF+1OtPDGy16i<%mOy!h9pz&bPkIV;?p| z)N}|+?W3Q_kLHmIxXc(1ILFLgw979v&B(=zh*QnmMn6$iMxr38Nlfc2&5$-Z(K0HHIV`uM};St%UyTu@2nq*L3Ez5yLqb&%vQTbppkk zA-(h=EF2a1{n_qhLs_@r;TBD$7s&+czT#Mz$w{E`s>dF2>QPy7%P~{cIlF;1*3;oS zaCU$XGWS?b>6%mz-l?URXgk<~p*njsTeJ?cvbx^0li-qA!X8(*Xo1-h*BW)>FvIzU z^XJ8_vScLJ_&_~5O0}UeXj4Q@ftT|ard1W`7I(ehC`qPuRnqRh(BEM-Y1EnNqhRLW z=s9QuI$RI8);9%E*=wq(*859j9OL91aBM{BEA0pg%upT&GUsR3zOpd-GF?JrH&81! zo`IyAk?jO)Ic3jXy~$s#MEgUVO{z&wg)Y0fHpzdIhKhX?f%5pIw@E(~5Kih~$3u>} z!!i2KKADm%3|LGqMQ(sgPC-4z;WJyChamoMj&CMwpqFwWSE4pe<{qZ$^#@3H;26Fb z)#YHfln~@N(<(Y^a}ryG4#@oHK(Z=$)+B>t!4#n--nU3Ew1F5bmhgDt7G8l(nm zjpmcaiR{^J_5Rly-8gs{4WH;!-;sd~ufl45^Jx9zC~+`XSA4btD)5B_reoW>UU5)KcX_ z1u#<0W$zn|uFd$Af+A!b2S4L^B4T5ma=4~3<2lEhi+iZ{eK#28O%&V}b!6~&|3{P% zv}~5umr+Qu3vXek1@xPMVwvbVaFe zw<%7w-Y$OR&EclsW!&wI`s^qntsVWv@)EKnD#gABx7w!oGs+7%;t4y)H^~yA?k9DPr>^KTPYElpH_Crh@EoHcV{fPepog-An!0>^eZOP*rTMksFq(R0_5GPTW~ebY+!vgCz}=5?pS__fQVmz; zHnjwSD2lmWQxZ*z^u}C;cshL4qJCk~m;N%!a5Tm4RPvt-C3HJ3Sn;(M>!ufOlJykz z62Ivv&LyQV>Lrq@;^ij9)!Z`{-xaDB5Wfh?BA-r$_bD6zLl)^E)@qT!@a7FoV2meFP!s5^TkfDn6E z!k#!*6K>Yg)L|3q5$wdt6REn3wT^9gLt5A{3Q@_gOXU4Jef+>t*gu<3qSp}aGm>(1 zjlNbJy6t5xoa`dA;rCwgX$z8X7O(&Gt`1(Tg{vFwUE;BSXNp|%K~B)bv^w!PnZ-V7 z{Ds^phTt@BRA-xfCK`y5|2e)tNm(%-cn5Li4>`xI2kF4ihRWmKG-GS|I*GHq8+0XI zLhOcowZ_UK?815*oa8fzwk&-ZYjmx}j~r@oJ5|fE^k@9leQlI7KOlg2^%<3w6Vn%L zoQpcqx&HOC(=amn?5*=EMD`{@F7{u?%*^n(UL)rogU&S6l#Gu8nD}dW!@Fmys)r37 zJbs)&!vqCEzDz?S5HenR0l_-LPPnKiGG-Q?BbA*zduV4-eN({HX%oX-Sc^PzWjY#Q z9?xe*IJ)JG^D_?(z@cGF=-D}VjU<*vM4V3br{fQD28UfB(KT?!t~Dc^#`b{=sZh93`a}+ke|*O|0qV4fWEQL z#)iN1c`LcLvRT-?5R>t@;*VshnK5s;i~=SjQ~90wmE}1k7ADQ4PJ^k0#TmFS(-!$b zF+xwSF?PZ%(_{C1Xe1DE{7ywv;Fg?py-3l>8qeI21hw{22C9Ew(Io3EY|s_Avc|_N zOd|Jx%>75L?K0rj);<-@>Y8y08DP;uWz3puJfA% zsql>VY5cyG5z%O@l^F+lSC^%n52PaBix#s~Hf_7F}bT*b-DWCVnPn(f%^ z0_{eJCCG+17FzfPB!M?#yp<>i%EZi3~RTX-^Gy%msTPe{7E&T5ef|Iw51Pif!m?fE*NVc6EHD7tiJw^R%gW|LV-XEM;xxAh z`(klvQPQ@ac0i8i$zJA^M)?sh(AD)%V(D~gdC8r2CePJGz5F>P8ek4CO8FU z#C$H^{UXeu9zo!78gU&5D)Xrxv!fa%nTB~cd<6Jhex;REKX@0MXPrHk z@krv%PRP#;9Y=zTz#y*;~kmU>6ASlx&6 zamhgYAc0uq@3iO4)(2KjlY>GHw|~`P@>(}sDOn#|%#UpHiLBK)6FPoG-r#lCe&8?F zH;|b>61O;TDLDxgiOe6OsM*w2sh78rnu3|FD z*rTt@m3V$>_wwLxgU{*wV5VC&kzx9-Zj&}PTtRRBu5h*5yl1#h8ISnBM@V{y$TZCR zV>E+WgF=7N8P*b{b@V*jO0QOi^QS6+?`_}en{fdV?!O2L%VVhILUip4-H#98;WZSZ z=VJv`N2feZa_6(>J85(*=Bb=OKj_{oRe zwndoLhGl(ipIM~alYvyc177&1Kw~d1ZCB5`&uY-5G4; z-*#@vn;pwpiWeqjoRFT#)PKMAX8bZ=zDWrvx9fs4oe(SLNP7MkM6Q^lfr}f%H&inO zr|KMiWd6ggFKj%$Px;bM)D9n3sY-WipYtuXd}dS@){}d$)42Y8)x9SJc7dbxIvRF< z#Nr82tQRIPj2wwOzJhcAx6npCiat&`(%^O5pJ}d#oG!f1cSx2H&rV6p%r(!N$o8AP zhI!i#f%vTPP239I@PuLXnK37Zw{x_3rEJ(1y+weEF2CZ3fr4(D_$VByX>SG+%9_SF z_1laRcO7N4dS0t_mt!4{G7>Du1``T~hLX;@A3f181#1h~uTKVNmSxad9JRTrlPksb zNM5P!|H_WILr;2k#-^eGe=s}7tiuLVr#4kw5?e7oRhyQPIjJgBRdl6AP3)opy&}B< zv0bf)#Tl&?ek_+E)=n*s0;|L*Vz@BX1}8YwDh58qd9#~b_BBGID*0BBJg^G4xE+jX zO?F~X#0hf;K&o{`|glB{=%lq5t;q{bY@L;o>bI?!jV(AV@G>f zke{KvT{O)m`*An?++y_Zrlmf&?NJ9wu}{w{&bL)GJ<5uZ>PHEa=IyO%D`DNBg&*@r zEb2~2y6mfQRU#j{AABc=DZeP%m&1+6ow|l@|9X6N(`faqniCL=K%hdSs-@`-*!av#6W#-vB$h zSG_XHW!RhVPst1cLR`*TAEM&JvB~q*^AdtD)wC1bXsa~cZ__PgMGrP`-pJ)ESSe7U z0Bt#r_Vr-5#f>jIpKo|_t(krr*m_fEIHk%MGKXxhs+|p$3bW{4+Eslor8yyz7dptP zGU`0J1!^~fm;|_0o>}S+Z0m#^Sm7vxF7!kYIKgp^0}@SLO67M%)*ICoHvK_+1p-_z z>axDdJMr=6-LLjoRZ};IH;*#&%7@7|NrMOi?u?BC0yYnH_J+4z*DZ>+uoc6J;>J_h z`lwzH)vFGoshDsWn#o(s_wnoIc!nmd3{L@WQaFGB@6tq)sRm)AI_s30t6q!~>?$or zv%XbD9^Lx5r}qJZ@>G@$^H$4z;XWDLLPaU5GW`T}uG;bUO)Pj#Hn#Uf$4!b}a)FV- zn{{v6-JT0n+&9pkzs7^&taX#wBwUO-L#%CA`j8kr0}v%^rmTj!+;C7*{C(>ZLs>I_ z*B#3&l0~GPX@~s`_ZL|e`#qLuE4RJy)KFZr!U{&`!ujbL@u9RrM*!iE{}%_g&II1 zML151QHOm!<2-?`@6sZtt&1d3fBfo(7=`9Yr97Wdg{XhM{W-hD9tJ2TOWW|zzI}5c z09MiwZL1HjGQ{5?%I@Eh~-h=(-Y~3O837TMbG$1 ztO+4g0)f3UHGL5Cq5{_3>{6lZA{n=SE{j*(UrRIS$YzmCK;2zkE6#fq?b!5*bZ_S0 z*NP!%as$+D7YP<`>zr}LvJ2qypV#p}5GW|NW`R4~*6RonH!_-Cv^w3H@1 zXWPo8%wZ4nc`Wq+EIdH8OPWadtWBAg#4SM);zj?#xwpPHX&_ZsV^?&_8+cQrHvhMW zUb6}Ut8dJ%@}+eUmB6UDL3?z85SyV=J+HRULAuTMV`{a2_^Xd`XX4=1W5OC`Q}>Ag zB12)vJrKt1y^GoEPS~Ek)pdI! zG?8&X7wgf#l;bxOO=Ol9nnsho+w(%38=}YSp6{q^Nsb<(f6@2-G&Bt|aB(t;aXs!$ zi~5V&hgQ;5aJ#`9;PXVN_$jDoP}HqI+&=3jC;hU|T!0bi+VX^JB+fVcoTQq~Dp?aU;K63#w-dQ?tzR zem2NW3r4}(wEg%(ShZeT!?%G3Jc2$0%x;phcZ2@$uBFzn9~aSheV57$#|9c}Q-ATo zGA7m{LqAb65R_UBa~&bKWU9~DKh`0jY5)B~xFk#H$bZ>j)>B#d#p%$=O zw-3g<$@S}m`-2!NQ|-CXv^A@yLS%7(D%^Y*e^vEiBD+W!K|x%8LsR`%#4YfhrmEoQv_0$Ba_svF`Vz%X-RLx&?AvbC;xF*mHfwb=m{3L=FHHe7 zpGb*W>-75ea*a+8v*kkR;Yam2|XEKIoDjDtsz?F8wc2??HnLuxgHBe z(6(V!mbQiTxDgL0Ls(n;JFfCls(|@GqeV@;t90{Up06jNsh9bC5@q3#$Pg2vIwQNN zNjyAD*3{bNBD{65;Qy)Mka{nqOaHka?PD}9OQ}@OkfwK43cULnEQ4R+;YA z7)3h$y;hw+muuGdbCjo7$5Upura6CW^Cc|VPi%6cd*qA4AbYBA9Xf6+7EMp$&ZAtA zjV00R;%!l=p=e+W!Wuy2I1Zxix%Lad6a8Jt>x0sjb;@`xJgTJJks{}p5+zwoE zG;JwE`9wm8uAR!Q9i8Ci@!j>Z1z>SDY5WGN1TGL{T+6CQtOL=CK(RI;ch9LZ2w z6ydkn}ievBQ;h{Ct(mDia|rzyB;s0SezZRLsLa;%oN*IV>s7Kk!Vh`wi=rp@|^EgXo>#C(;GsAp}KOypN&d zLsJQBuz;~#LY&kiK)@F>t(hBS2hVx(+P6UH*mTQ&nL=#f#7_bO6fpzS?X8E%Cz7O> zL0Y-BpWiTmd6qhb2H1ULk4kpu0~e{6q4?mfB-n@~K9tP2_ZJYo@)Drp5+K#5fj=;{Ei1F&xICA)5&f^S4b~7*h!z=^DYa+ z{^sCIqqE5xH8J}X9yx!nBjP3Oeao@C00*2Lota%fYzRXY@={s#m*27DZ5>L*HB zP^JCkJS)rYv|^)71(~BrN3y@aT~+jN2V(ye_4O}UVCiVN47MQk`6}^`(l_USeA&TTV!t<=x7v;7P{!RR6k=TW$y5b`Rj z$MSOI7k+-rz0)@kaC>iE8A((EHfq1|M@i?MvIB3-)}cg@wum7 z#Qu4{U z9ig!gqW_a*)r*=4*;UXxXc+Q2?23A6(QXP-jtVXwD5X4k#)rCfcjahEPL6qLjCBAjjNZr7no5&%E8j~=Mk0gdU%W>=^2Pi()r?}kq=iu{ohKjOG&Er1IAMi10kWO z-~qzA&jEgHVX0hy52jbFsmTNjv5ospK05We-a~t>oD9>efE!N6un!PM`#yO8m|@-j zEe$|*ep4xX)=|%@5YeR%#b)?Mrh*=dS?1VZxvR7vo#dQYd?f3ttM^kIE6)c09v&q% z`3R)N8Y=5rR#CFS3rbdueb}ADv9!!p$_lt%;ukiUd%2bO1WTg{X7CB`*#20( zC^t5z&PiPI3{JFtN4XU8Ge`A?e@&}eN`su!R2=+`3 zTwnQ+tV2EW*Fg`mCr5MEf%`q+GX3(l$S(KFF_{VaX-Q1|u;~kFCGG`4aXwDsnnMhW}tIf zb^HbND~4B?qR&D~tq^h@0ww5<+&em@YuA7c}#{2}i zvJ!h;`JBLk(N9EWy^#Ps(V0tdqYgb<0%N+NoifZl^&+mBh}sj1FOa8P5%_#aJcQ z*UXP%^9LTuf`5O3)QkJr31yNBpCYG6BZEei509ddhY9qL+3-Tk?8ZY97)Sg1(ov}# z9ZU~jHT2rWo3%9zC{!W<+n8l8^5Jm2UD@ri z7NQYwiJuY#2aSyG{vWR07(3=YsMdSQ+6<&(%{OEE?6PuzVEcR#_`HY zRgj*t9Mt!)Q7dQ`_@i%ryjd&bL&cV0{qQi_&;wU775R!oAzAy5E z&weFq7Qb)SqBqt0!z=s;-O7I=?_-}2*r@QQ{B(J?nde^HH4?baV#v{oWj*l--fTYT z{gm9XwoZVq40*fNsQy?riP!gPstK1yf~AN~-KhlHN;%cf%McCSYBmUR%HF~0|Jio&7Pu_h)iGmgDPBG1#1oa&XrQ~y`&=VW8>0o8X((hRZ8uh z1a!nLgF`6ajTO?za>ZNSy?5^SiM)5{md}LkysqO^K3}__Rl!doh4-*UaKy+oAr;r@ zDOE8HbQBPYEAg}8Kx%g48A=n7El|d7va6P}A|4~T@nnmSqutVjv{XngSB-{A;fA|- zR^jH6_ob^I!#Farn%HHl)$JyNGt5Noe)qYJGHL9WPa>o{L5-&{7$Dj#Kz8$eiq`lz zO@u^6N5g@OLX_Km#IE;^#rJs~e<+A|GMzXAC?+0)D+*v1Z&j^`I?V9C$7L8VsTWD} zg{L&VFA;fF#b0wB;&TttN|R87urGZwAlA-W1j>r3bO~)NhBN(U*Smq;&yfPjP|5^4Yu0SO?em)=52=)DPu zD4__UBTc&W{?5JQeShE2_v>SXF*0&a_St*wwbz_;os+fU=y)?ixZb^+Xsxsl-~@yU zopS;lz0@;J0j6F~`*O*Bk6vdbP)<{4{|8s3YwDp@^h%gXF0Gc8+*bx`GB8Nk`tDm@ zw0xlZ<6EPMjUW4P*<01+7=on_>5?TJ!n;TBiqA>b)3h*xoM5%3Kc*xiUucpJOSazG z?2Wn+#`EUtsT8;GdF@{PDU}tfv%Hhlls<^-Nq1IRVdlZ+*)l8;v#Sr%`e39Cqknqf|YKy0G-w`T|j^_m~*7sgK?o3-1X+ zVM-P&rK*=dq3n7gv5LwVo=dhH;2k0^$mu<(n3yu>P{{vob+607>a@}@c&v_Vc`@iwY39yiUCUH z1q(bwzk>1Aw+P@ChROIgFxJo1)@NF*DF+EZXPPZ2H#A!l!G%Jf{ohhcEL4c?pzb|L zxF`GIyYTzhA~7TU2HHJQp@X4*-HNeMB6(EG$i(ghPTM$zRC{-U6ge}y=I4TGtIy%h z^m&fmP2Cd@)&#s~_K#Ur&rY5fX!icY^;&v_E$oPdIP!8>^+H6>Pb!ASBFZ63(1}RU zeSwzsCX0z<|mLR`6*ke#(|PxbZI zDt^l>eMSI95>P=%H*yU3W(XNE^N zKSEk}$w#w-ODNdOE=jQ;e>fA1L}ff@jp@B=*Z!>SQxa7IpZv|XbZvlr!qa3S9MRhR zIWT6*st&pQGi0Nms(q!tyJ+1*G50Po9)f?za}8{XoO&|=TF)qX30M28))4pIQyRXc z7Gq{ns)PrGTzN=?$K`k&0bc!bS(*a}CE={tB3aHq6M||G|MIzbp^CGc*T=JE<32~_ zf~GX%3Xg*C3Ne3quF|_E2^2=+_dO{9Q<>ZqF>hXzc7VS)f54RgtK6{QE31CeQKCB= z4}4!s*Gg4Qfr5Gud%_^9k^cRu)SXTzAxXcZLFRbrEA$l!A9Um|-@y6!5+pas`Iqs7 z+ExHqf5vS6ZLrJGc{=sw_BuOY9_2Hl@jIA7oCoC^PnCZh92dSZm)}SXqAW$<>-2>0 z#!>t8vvpGu3l67^snogfUH$2DD{#*O@_*?J7R?vvO_rs1beCpv1a|&z87EySKe^4y z>_dm%zjn~j@&u5h<*BW-xTjDT>mhjCO`R<6p44Q8Cz>1zZSk5EdVnXk=V<-UgyxDP zn`uCds-u9KL$zulw?`fMY;jwHz1+=?1}d=0`4j|)3-B@)h_YvJSr?51-iiejVZC#UWU*-TNMxt~V8O7coZ-Vi!xPc#G`G z7KB!n+(>o^l}72&>RpdM%V=d8N}vD`o<;R2oYAr}CgXG4jF;Ji3N>o|l89Uf8tWVe zInxO;wAYZCK$ZmCQ7);vWN9{hkWB zKZGk9dpEsA1011Ossw61{zP+v>1>pm%Yqw$#MYHZtKM_xw6h0`l5d6RO>(`_o<0er zPH5yGzk4Np6Yfpq@UeDee#T1tj`44upf7}7X^91sB)nV|YS=hQO&PMKNdq{Ti$E9du9HiX1iW4yle8Sp#I{8rWd#)WzvTwR#mlz z2xAkL!>n#y(R$JIj|PPasgM@)yOQO}Qs>YL%pZl@oRavudirzorRHbNcMbgfetXwJ z;GG+#Ht6NUEukLbR;Fe~??RIM^=I$MY>mwYFA0fX2F>hWC#cltm?U#s)BHnGDc{Gp zKE?RZpb+>TRaxKqX;*C)Cy2PinMBx;Hb-!+FpSfV#X0H1rXMf_7O$Qb-zYS7^iHL~SFAF?; z<%W|{S`x9IK%xY@Lwtx%9K)smv@=DKkwKoF5FMp<&;544>!Jbb=n z21vcxQ>*p6RMs7({Jlo&!qz}QpI|U`&j|41Y~MH;g>5gGaA^+Qdp%JIJG0A|+rGc2 zu*V15pV20J(>ozWPQ>l>ATpz>&;Uo~x3`kBbUwS=d4cXOqfwD22I|Vp*l?6nTqiew z)|CnjGJ_Bi66@^pE$bE)Q#pqpdQDbNuT~c_T2Bl?49v_0F}d|SwK$`B!+EfW^iMem zjWg5R5BX7Nt)a(G6(8`DDDB9TQst?^oauN0{=xQH@?I*2Q%l$nQ1F2c~J2*KJJ=*hzs~wW?C!6c0 zq^~ecy@lxbPH{BuZHnxlk@Xmtnq6B6Y0bN4rw2Q7=6ooTFAH!oM}9Ca4s3OLlIxJh z67IwNFAU8Z!81tPizZvm?7W%lTogH;Gx>d%A^&P=zn)LCQAA<6`@8nrHp&TR&qjM_ z5KrV?Diyhj4|#R<0{EABj4Bni0MhH1S@)G{U9Vq`pe-w`!*^CkKfSr9KuJ)T z@0VW{N0r_B(u7pWj2#N<`}HA}zZKLk68sZSn68!pIrlXHS716(p{qQ&q}FaR(SC3+ zL%KCvJ(MY9vx`8DX$nSL|I2*vX9oi4zF=zVX=J-Lq`NjYcV@o)4kMbZFt2*f5FxD> zSSc-}`AVyvvA54mNxa&%hLqMq@Ed_?r#*)L7Y%Xe=dK4oybSx_B@kh40emDASt;EC zoaF3;?B+yf+}(Kwh)78g|8ZStU)n&&`F*#Kmrn>5^+6@7q0xp;1*kEb${7ZW8wGq# zM$q=0>>aQC(XN@~P$mwV6ALeBSWekfcz9L7g?lL5_Cc-bLxFn8sdeRJ2>kvenYP1c z8M>Mi+4LL(MM|@>)tUVTO_%(7lDb_qKMhmqm&+J%L+?@FrQ>{S=nov@&;2UAw+bbk z7sOd=7puhGpL@V>XzHD$@X18-A}!<&-fUT{|0EO9NR!Y9D9pRBuT(lt(wvG%jE(wg^he*e z;BaC|h~Hxt@cwO}xiNgH!m9oGD1~bC#j^C2 z{rT|KOxyvm6W(0_=H=sDiNg$)mkf%-e}4-Y1x0n-iGrG`7Y+V)BOZzYTY&JKtC_zO zP2L|}Ulbb!N>JL**s!h%ne!G%sZg#=B<5-f*(%(xkopfM+~rVTj*cdJGk@YN9q^+g z^FMDa3zbh*n9<3Y)!&h@H$z)%OB1@ z?R@omjCARAaoQmyYm%>+2K6^~>1}?SsBu+zv8WNyWuLw7xKj2e`~x4H>L77X(zSnW_re}dKMwb*L;T-5|E3v zXivb6#!-%NygcK|?rA);CZ%Nab6-O}h;yadlI>hN{BpkLAmHbzV%Ju6FUh=YqQBPK z+z(MkpgrM_NF>lM{4;*T%Qo<#M4^UcAR*Z@58}}+-Y;m*Z!Y*y#)K&BGihN0>2}Vd z%2S44iuSLSS4o(v-79|mN@~+Cl)$4aq}4|xPxM;uJg@LZq-H=@R%=bNOPFpX##g^C zN$1Cdd=iF!FqGl6vpA61)jzD^Uge)jMx$?t_8{i$2o_48iOd?@-BicZdKauJ-wVh@};He(dO z1td+iM$lcnJM@f%#n(KKn7z$HXzX3d_0Gagj`8_fk}~fXN`M*%BbYjWwXO^XrS9OG zR(_70&K?tjDsX|LdxdtL3BGVbusC1Hw78r9>#wrkRqlc^3&E~bL&Az_NrP}(B?ZtE zKel$Da(j8D>!zHcZGDAwP_{$VYx-RewR{)#)>N}@-KQ!g?*w^C3KUMBhBPR~(jCs5 z7~lHJ-8oXY)63Sg=}*4lPhQ-xY~~a#wrf?{84`p))J*X{;hL>2>UVTSD%nFUontt9 zR5`(Zmr8~v=`=HW{+DJ&wcBStj+}?DoXx)!7keJ{3)aj-;5ikZ;M#V`gUP9<($3=T zuEdN+SI%s*oxh+=`?Mg0(g~;KDxPB!0m}dE=ftSDLklwY%2l2RbvODR@ZxsGRLtv8 z6F#CQ6!*IKTI;`eGyt3bzlUQ9!N8$M)SA6-JGuY+ES`xL^6%-7sA)9tlm2bvF1Pbz zN1FdW(yp6(t1Ju(Z2Wf*Ai9jKN6J_Q-ujQMg z^u*eZBtfWOK7-%Q%Ps8YMSL#T^+ca!C-knv3*OlYUTskEyg+A5;PQ^{kWA_OY`d!zP!M}LB4$3KhM}J9XF54@fJnwYY$$WH}p9lbb+Uf z>cW!`DbZv@?Y)>C<{inBhNOngAVkdy$dMX!2Xt?+g&Kq*=nTkoQN{;FqDbMDB~jne zTLy?4*)MnXBUWN57BoKD#d{zVVmv;6XHp#5*&+vhBL)$GPVhjHzRzogw<2{h*IqbG zl4KD58x0MI-^F-Zk0F^sE#~~QaomIug|>I%CmqHyz3w%QuR=o3THue`RR?j_IAF9w-AVfbQHV{V zOXv+P@`$(#N_j*wzP~=ig2Pm^)>LrfFm0CEg3U5F1kmj*e*D9YehStUz)&$lU{ySj z8%Ry?*>L08O`eODWs`GxqG{k;U^oUKKlgGL;o*y1bN(yEL~3BVbeN&Hueo`&llrz| zW6m0(H~~Z@nlt4xudk@a6~gC(awMjS)mm2hjWLEDt1^ zaUydpVBJFSkwm}f2IghrhobXBeVNNC{^$?>5QXbkCbs^|2WOi_9{ro!XUt4Yi()Mz zz<|mr{E2o0TnasqSuVBHA_N2kZeQqQQh{&w109WHh6dwxUd~Wk<`RNh)5TB9fde7D zIpKV|kMw1)I6|0j4F;KPWaXre2r>=b)-J5>y)Ine)|UxO^+4{-{$ht0Vu|rO7x@32 zY09FXKKdp#l8-FP?lZZRZkcnyN#22OA0{?mUNT?aBb_r6H^8#7u{3jM>)LHm>KVXq zo(Uf31D>S%=Wp0c?Zt2jIjO)r4`eVwCkajc(rS^%ybN$*GOQG^eUmfq2VzjZE247r zSzW%g0ro?=o0H+WV*&!49_NP~t$y5r9G;b(PwOiWB?VZ*gm0$IZW1D4Qt|Cp9O zm6+Zh?FSd1ZG1kwxcrSgBFh%_Aa3 zuzw)JyP7nfjRb!bKxagH#m^lP%AE^IzC*GPo|Xc$Dl(LdCDbc6dWJ~QAXBaN2?I=v zj?+@&xE0nb7ONwLi%3p?2e-lo@rbYPgJ6PkR>8)iS?7fDsHwN$Ov8KysnTYNu1h`>SX){l}jH zrx}_LHe3s&kF*Qr z4_~{A29Zdt0J7y?5^2C7^ zPUz;5kY8;R$*GGo!j7A5Rt1Mykj5@27GhzU)1-SXJ~&HjI6h;#WvCOd9FU{)>e;6R z64Sz`+0CEdEDu5TRI)BemUm+-hgc@Y(*CYV)DI^Yu(x2<ndhUfZ!CHPK!mkl|is?S)X``!W##pHO_|I%_jMwDuANp+X z%%l&gVSPBvbuvse7^~lqREQmjALamcCR=j3ZOZ zav-D|&1e1m`E!BiX{o;vA+o4xNNQwn8l${oc+LScc?>Ksz~SN?+6u)~y4^h10-Cmr zW931SkAcwWv-CJZ#o5Yk$!*2Dn&tBo`fcOj-H!X&x^0u!7k^_s+9^O0$)w0PjXmZL z-iKe$hF6CHl&NRzjElWwRveZZ=~xfvcSHGHlAYV`zBC()&{l=62MlYm>V$W=SUJ39 z-1se7!3?!60ZPz9vZ&@FoiJx_zT zgnp5Sy_o*{Umr$SRa|i=h*5gILenc3Uc=1BDM9*PxL1?=Y18pu9 z>lU9-J^&BM=g%6pEU0kcz$Hkm9H&fD$PAM=PK%+yrk z*mFYI@x)~67LFmvxArj~iUdTtkpOV!4{iz%|5dIqz-|vQI;&)@d|nad;@`Z08U*=# zs(MWox=%#e=de24?1VdM_jQLlC`B~4NIjvjLTT95t9DtQgQMra22-oSM;y$@t=uL&{2ng4)5G@4H|ADLzLj=jbct+F8fv~Y0sv?%A9 zU{GQsFTw|>x)`Q3I0De5IJ`w8^&bhsc@cB^^-T=k-s&)aiK2qzFyU!FI7VxD zcf?NcYT9F-`6&G3t8qd4k7NDK^htfuKDb*1x75S_84m(+qO%iMr+}QIb8C7zvg|q) z$U%GiRFP?E1U*-pJ}=#Ff36O&IHvkm{EcjP1ha;3>!BfMrtdwF5R_w=4jd`7H&>@4 zqzOQo7;xTYtCK+gua4f81Iw)8RZ?B0HxPJtewK(s#3LXmn^B)}I$&T;XrKwQx}dJU z5T*8z>A$P8iDDZsnqWIAOG`$wv*F*YUT5eR-iT)-#|`OOeQ@ot-rA~qpD_A9 zLpm7q8-TAwRLkscnFxc?IlYv?wdDZLAlcQiRt*-j>XzN!X(l?hpKU(A%=Uhn=7C%| z>IP`iIwNq*vgkIFUn8q0P7W}^cej)T)^+>i<;lwYjiO93cBV7r;^=AC($bPcfu<;I z9Y51BGEMx1ShhqqTQ|Izf^5|LHwv|7Z z^%l!-xxr_BnDrK4sGjqgtcFI+2O=Q4XFnA(`6e|s`fC7AVHh{3r9}XczlGj2-{ZdM z=4!%xtG|lk0F`?FO{VxXM9CLG)_mRNKl4D4Rskjr6^PvHnMB2x!K$BCs4sDVsM!~r z4mT0=jO;~u=xKX(F3pyhIsHf`cs=hnCHZCFE0|sH?puRwx`-wo)I;KZE}?!VS+{u) z}XIuC7v5h+#W$ov<@Cy~*J? z32wokehi69>diE{`}0Y6l>oC`wiD)|8&ReYZm&jIh;I4$*V>A7;bDb=IbKb+*f1}E zg90KT?hB~`fO~sjw!(Aa{vS_M-LPaoF+1$A8ZD<)KVOS@v;N(Bj+gMdx=8|V4LrP058dXce;ioW z!FPB?p{ST9oQ55K&E=Itm-B7Z>b1FG1}P8_m3tSo0apH1kUpY&%4Awx>D#Fr9ee0G z{>z5HZ~~JD?A=lZCiHWF-zs)bN;zA>5~7aPf8b0@#~<-KWxX9;-9`J@*bZjf?D;-4 zIE?c3k!heI$E+VJ!E^sI zC?Jp+oZF`CB?VPe2Yd_kdDJtu`7{P@$R3)pW3m59cCZ6@ypeZdo*Rc)zJMldLD}CA zx<-zT*wi0M_7}6sf&|Rd_7vCIsLapOkJB`t_;LTB%P9OQ6Fl}Puh?yQY44&FC}2UKV>G zKUe<>0>+>?H)&WRwsvL#qK>?-W&$J#=L7O6@%BWLc#Xs)fUC0N6>Jf@5YDomJOI4L zrO4wWwKfVEPlvd{q4f=G3P6hvG6qaOUn5LQj|ODGrnff0F5S{31eNv&AxdcP^S?wJPgdRi6-rbYy_>W4GG)PLRf%x3kUSq zWsKc?s@rDZm>|6ZCXiqO_I~2JTItn&>*7^|A`elNk4?+B9(^e8#19ZB1%NaP%{0eK z-HHQ5RUV+rO*S*xgRVw^W2we$UX20_$mny7U353G*w&2!NpRPgJReGUxH);Ep_sI1 zDck#`p~#~>$DyO_NH~wf5n<ne5`-;O~yd zbh^}f#CO`qQ}T@lF_CfV#owQDWyQ?=>zL7-Q8bax zN4(W7Y&R1IufI>H{s}+*=rWvj`)pu;-wj9&=qntG>KB^275o5+wx2BrF*?8wk}l|%mcjxk z$;tPRNvkQ-1vpak`=%OLM8MVT?Ch(V{mFgK{z@6T6i0n4e7xy{b-=7&8f#OmzBpHe z)vL2Bh5|VNBItI26DeSA>4y(di03na7Cpb@1*Z`{JP4Jp^kx^hWNikzsx3z+3I z_%WU>b?U(R@bxw-i52KGl^Ad#Caly#r);klNl{!^ApwDp6<7KoPjE&frI!9X071Qg zG|<_Fw^M_*ya3=jzmE?We;5aCc6L64CFIYyd0FbrQl0rS0I7iTLXoLJwzv9)?Ry-l z*nVlgzv@l4_Q~qSS%*Lyy+5W+ z1hVEdYLn^}r#d38^;;dhaVo0$!$Fn#Y5@4Wg(dg`5U8q6f_vuYz99|T7btOX$d@oU zw8=x^f8zm!1CY!4Ar1PmnIJOYRy(cOm_K5>)$C+i^(u6*4w~(1&F`w8&x^(I9D3M#hjBD*h(I?y{&QMw~8(5 zvoXOtW_=9fVw2pU__fniKxw^*?9YyU{>Kw{;`Wa#n=Bo!9w2r+P=r``8aVn^Ie!?= zGWkK^pF)*0z3mngslM6@J$NB`=8@zH&3kuB0GU)4`Vrlw`!o77if$1#Mt1<1o5k$> zp#1T)&{p*Jofs8%P(fRGYs~{ma{BEny^cb2eEDIM?J&FEzhao@EF^+Ar;QFY4~S4q z_yb)w8br-#;yU*?8D1xV&Rlkrqlf_QIaRt6{|;a-dCvUKGg&?mg@^m0gS3HeDp@(0 ztgdH%4R!M7UoZH4GZH>~mm4qUpl|vweUWSH&7tzigWZu@HkrqUDxPyoWxk^Tlh^)q z$qe9{YH@@zg-)AWVBirqJ&X8khj%?>`{>)UIABA-AhwX?n@K>_c+e{hPgfIVdLYF! z4M*KJk3P-5mQNpv2mo9U3cB}PEZ{WW1?{y<0i;i|KvC{QUsEBI$tD><77_+Ie6-i% zfn*X;W095ZGWo-8skhvZ#A)6>=23az z>+OL%Sv9%kW)=bOJqBXi%MQ^kH^gZDx!8NuFv=&4Q?Gb|?;L8tim1scR9;URdRBLT zr%DgpTSg`Z!m7{+pMvkM7dPD*c>OQr^BWa%x(rt)K!Z|Jvps z*7o)c59A(@_nK8W4d(CqEii)#Hw1JXI^UhKr$M*nApm0DnVfOJFCE5vuJt3{kFMH3 z$Gv=%<4L+t^vD29Z^@4V#O`m6v@c)2v}ddD=-=jf*hW^6xt1_HO|EAPrX_j=v*yIY z$1pp?3Qqn)S;NcHK`eU3>)WKOSB1Gib5BLHzU;TyxLHk>@>GrQUq1niy9qpG#@c0+ zX}7C)6@(S2Qfj<^4diuIJa97kTpWFW!LXO-{&lvtJFB=afO5XSR@;bA3(-B(M+*rL zzgh)Oz4;zQzjyZ# z2V%R7@2~xT*E2`qM#LFnBo0u_7l3}d4~+dbPp^(B=o^}=dk~^<_DUNH3vdVhQUHL` zP8eM5vh)}$6`bIM5#F+R&(D1(rjC^|`l|~?1PzIYrUr6wXD0yZgB#KG5tVzwy$bB@ z#16T?#}8h{dly+NajZ*fb_fzGPnAww>Lm5GFhQ3?KRRv(rke8yE^EDn*Lz+e&V{Yu z-E*^i>4jLbbEH0^20BO4(bUm}KlKU{c#ZkkAe^)pbQ%SGqnEU5X^MST!DR*(;f)h* zsw=2=-3;WNp9;2?$WZt`XXsUfiV}*nx0ziw7tP{aNqXDdUTg;sQ_4C}ZTHdvGPysc zMl74Aj5@-U%uhqkTAXrxaHAQzg<-CK*WHmmP#0Kg;-%r-Iy-T&x!>uPAJ<&4MtWtN3{y+XKF0cRq literal 0 HcmV?d00001 From bf48fd5456591ecb34023080eef667627bb74641 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 28 Jan 2020 08:29:25 +0000 Subject: [PATCH 203/257] Improved documentation. (#2053) Fixes: https://github.com/thanos-io/thanos/issues/1389 Fixes: https://github.com/thanos-io/thanos/issues/2052 Signed-off-by: Bartlomiej Plotka --- CONTRIBUTING.md | 46 ++++++++++++++++++++-------- README.md | 3 +- SECURITY.md | 35 +++++++++++++++++++++ docs/getting-started.md | 25 +++++++++++++-- docs/release-process.md | 2 ++ scripts/websitepreprocess.sh | 18 ++++++++--- website/layouts/_default/baseof.html | 1 + 7 files changed, 111 insertions(+), 19 deletions(-) create mode 100644 SECURITY.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ea038e5b5..2aca8dd5da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,10 +1,8 @@ # Contributing -When contributing not obvious change to Thanos repository, please first -discuss the change you wish to make via issue or slack, or any other -method with the owners of this repository before making a change. +This document explain the process of contributing to the Thanos project. -Please follow the [code of conduct](CODE_OF_CONDUCT.md) in all your interactions with the project. +First of all please follow the [code of conduct](CODE_OF_CONDUCT.md) in all your interactions with the project. ## Thanos Philosophy @@ -17,9 +15,26 @@ The philosophy of Thanos and our community is borrowing much from UNIX philosoph * Make it easy to read, write, and, run components * e.g. reduce complexity in system design and implementation +## Feedback / Issues + +If you encounter any issue or you have an idea to improve, please: + +* Search through Google and [existing open and closed GitHub Issues](https://github.com/thanos-io/thanos/issues) for the +answer first. If you find relevant topic, please comment on the issue. +* If not found, please add an issue to [GitHub issues](https://github.com/thanos-io/thanos/issues). Please provide +all relevant information as template suggest. +* If you have a quick question you might want to also ask on #thanos or #thanos-dev slack channel in the CNCF workspace. +We are recommending, using GitHub issues for issues and feedback, because GitHub issues are track-able. + +If you encounter security vulnerability, please refer to [Reporting a Vulnerability process](SECURITY.md) + ## Adding New Features / Components -Adding large new features and components to Thanos should be done by first creating a [proposal](docs/proposals) document outlining the design decisions of the change, motivations for the change, and any alternatives that might have been considered. +When contributing not obvious change to Thanos repository, please first +discuss the change you wish to make via issue or slack, or any other +method with the owners of this repository before making a change. + +Adding a large new feature or/and component to Thanos should be done by first creating a [proposal](docs/proposals) document outlining the design decisions of the change, motivations for the change, and any alternatives that might have been considered. ## Prerequisites @@ -27,9 +42,12 @@ Adding large new features and components to Thanos should be done by first creat ## Pull Request Process + 1. Read [getting started docs](docs/getting-started.md) and prepare Thanos. -2. Familiarize yourself with [Makefile](Makefile) commands like `format`, `build`, `proto` and `test`. -3. Fork thanos-io/thanos.git and start development from your own fork. Here are sample steps to setup your development environment: +2. Familiarize yourself with Go standard style guides Thanos follows: [this](https://golang.org/doc/effective_go.html) and [this](https://github.com/golang/go/wiki/CodeReviewComments). +3. Familiarize yourself with [Makefile](Makefile) commands like `format`, `build`, `proto` and `test`. +4. Fork thanos-io/thanos.git and start development from your own fork. Here are sample steps to setup your development environment: + ```console $ mkdir -p $GOPATH/src/github.com/thanos-io $ cd $GOPATH/src/github.com/thanos-io @@ -41,7 +59,9 @@ $ git merge upstream/master $ make build $ ./thanos -h ``` -4. Keep PRs as small as possible. For each of your PR, you create one branch based on the latest master. Chain them if needed (base PR on other PRs). Here are sample steps you can follow. You can get more details about the workflow from [here](https://gist.github.com/Chaser324/ce0505fbed06b947d962). + +5. Keep PRs as small as possible. For each of your PR, you create one branch based on the latest master. Chain them if needed (base PR on other PRs). Here are sample steps you can follow. You can get more details about the workflow from [here](https://gist.github.com/Chaser324/ce0505fbed06b947d962). + ```console $ git checkout master $ git remote update @@ -51,16 +71,18 @@ $ make build $ $ git push origin ``` -5. If you don't have a live object store ready add this envvar to skip tests for these: + +6. Add unit tests for new functionalities. Add e2e tests if functionality is major. +7. If you don't have a live object store ready add this envvar to skip tests for these: - THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS If you skip all of these, the store specific tests will be run against memory object storage only. CI runs GCS and inmem tests only for now. Not having these variables will produce auth errors against GCS, AWS, Azure or COS tests. -6. If your change affects users (adds or removes feature) consider adding the item to [CHANGELOG](CHANGELOG.md) -7. You may merge the Pull Request in once you have the sign-off of at least one developers with write access, or if you +8. If your change affects users (adds or removes feature) consider adding the item to [CHANGELOG](CHANGELOG.md) +9. You may merge the Pull Request in once you have the sign-off of at least one developers with write access, or if you do not have permission to do that, you may request the second reviewer to merge it for you. -8. If you feel like your PR waits too long for a review, feel free to ping [`#thanos-prs`](https://slack.cncf.io/) channel on our slack for review! +10. If you feel like your PR waits too long for a review, feel free to ping [`#thanos-prs`](https://slack.cncf.io/) channel on our slack for review! ## Dependency management diff --git a/README.md b/README.md index 3e4446f9ac..71a4c9ad0f 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ [![CircleCI](https://circleci.com/gh/thanos-io/thanos.svg?style=svg)](https://circleci.com/gh/thanos-io/thanos) [![Go Report Card](https://goreportcard.com/badge/github.com/thanos-io/thanos)](https://goreportcard.com/report/github.com/thanos-io/thanos) -[![GoDoc](https://godoc.org/github.com/thanos-io/thanos?status.svg)](https://godoc.org/github.com/thanos-io/thanos) +[![Go Code reference](https://img.shields.io/badge/code%20reference-go.dev-darkblue.svg)](https://pkg.go.dev/github.com/thanos-io/thanos?tab=subdirectories) [![Slack](https://img.shields.io/badge/join%20slack-%23thanos-brightgreen.svg)](https://slack.cncf.io/) [![Netlify Status](https://api.netlify.com/api/v1/badges/664a5091-934c-4b0e-a7b6-bc12f822a590/deploy-status)](https://app.netlify.com/sites/thanos-io/deploys) +[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3048/badge)](https://bestpractices.coreinfrastructure.org/projects/3048) ## Overview diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..70cbeb48ad --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,35 @@ +# Security Policy + +As the Thanos team we are not security experts. However we try our best to avoid security concerns or to avoid +writing features that handles sensitive information at all. + +It's worth to note that we assume the metric data to be sensitive and important. +External labels and query API parameters are treated as less sensitive, as they are logged and put into metric/traces. + +## What you CAN expect: + +* We follow best programming practices. We test heavily including e2e tests against major object storages. We use vetting +and static analysis tool on every PR. We use secure protocols for building process (e.g to produce docker images) +* We don't log or put into our instrumentation any data that is stored in TSDB block. +* If we use crypto tools we always rely on FLOSS and standard libraries like official [Go crypt](https://golang.org/pkg/crypto/) + library. +* We always use TLS by default for communication with all object storages. +* We use stable Go versions to build our images and binaries. We update Go version as soon as new one is released. +* We use only FLOSS tools. + +## What we DON'T do (yet): + +* We don't encrypt metric on local storage (e.g on disk). We don't do client encryption for object storage. We recommend +setting server side encryption for object storage. +* We don't allow to specify authorization or TLS for Thanos server HTTP APIs. + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 0.10.1 | :white_check_mark: | +| < 0.10.1 | :x: | + +## Reporting a Vulnerability + +If you encounter security vulnerability, please let us know privately via Thanos Team email: thanos-io@googlegroups.com diff --git a/docs/getting-started.md b/docs/getting-started.md index 889d58bc61..df31cedea0 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -98,14 +98,35 @@ We also have example Grafana dashboards [here](/examples/dashboards/dashboards.m * 02.2018: [Very first Prometheus Meetup Slides](https://www.slideshare.net/BartomiejPotka/thanos-global-durable-prometheus-monitoring) * 02.2019: [FOSDEM + demo](https://fosdem.org/2019/schedule/event/thanos_transforming_prometheus_to_a_global_scale_in_a_seven_simple_steps/) +* 03.2019: [Alibaba Cloud user story](https://www.youtube.com/watch?v=ZS6zMksfipc) * 09.2019: [CloudNative Warsaw Slides](https://docs.google.com/presentation/d/1cKpbJY3jIAtr03M-zcNujwBA38_LDj7NqE4LjNfvglE/edit?usp=sharing) * 11.2019: [CloudNative Deep Dive](https://www.youtube.com/watch?v=qQN0N14HXPM) * 11.2019: [CloudNative Intro](https://www.youtube.com/watch?v=m0JgWlTc60Q) +* 2019: [Prometheus in Practice: HA with Thanos](https://www.slideshare.net/ThomasRiley45/prometheus-in-practice-high-availability-with-thanos-devopsdays-edinburgh-2019) ## Blog posts -* 2018: [Introduction blog post](https://improbable.io/games/blog/thanos-prometheus-at-scale) -* 2019: [Metric monitoring architecture](https://improbable.io/blog/thanos-architecture-at-improbable) +* 2018: + + * [Introduction blog post](https://improbable.io/games/blog/thanos-prometheus-at-scale) + * [Monzo user story](https://monzo.com/blog/2018/07/27/how-we-monitor-monzo) + * [Banzai Cloud hand's on](https://banzaicloud.com/blog/hands-on-thanos/) + * [uSwitch user story](https://medium.com/uswitch-labs/making-prometheus-more-awesome-with-thanos-fbec8c6c28ad) + * [Thanos usage](https://www.infracloud.io/thanos-ha-scalable-prometheus/) + +* 2019: + + * [Metric monitoring architecture](https://improbable.io/blog/thanos-architecture-at-improbable) + * [Red Hat user story](https://blog.openshift.com/federated-prometheus-with-thanos-receive/) + * [HelloFresh blog posts part 1](https://engineering.hellofresh.com/monitoring-at-hellofresh-part-1-architecture-677b4bd6b728) + * [HelloFresh blog posts part 2](https://engineering.hellofresh.com/monitoring-at-hellofresh-part-2-operating-the-monitoring-system-8175cd939c1d) + * [Thanos deployment](https://www.metricfire.com/blog/ha-kubernetes-monitoring-using-prometheus-and-thanos) + * [Taboola user story](https://engineering.taboola.com/monitoring-and-metering-scale/) + * [Thanos via Prometheus Operator](https://kkc.github.io/2019/02/10/prometheus-operator-with-thanos/) + +* 2020: + + * [Banzai Cloud user story](https://monzo.com/blog/2018/07/27/how-we-monitor-monzo) ## Integrations diff --git a/docs/release-process.md b/docs/release-process.md index a535c85de4..988fc22521 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -5,6 +5,8 @@ menu: thanos slug: /release-process.md --- +# Release Process + This page describes the release cadence and process for Thanos project. We use [Semantic Versioning](http://semver.org/). diff --git a/scripts/websitepreprocess.sh b/scripts/websitepreprocess.sh index f8d5b45273..c4bc395498 100755 --- a/scripts/websitepreprocess.sh +++ b/scripts/websitepreprocess.sh @@ -42,7 +42,7 @@ menu: contributing --- EOT )" > ${OUTPUT_CONTENT_DIR}/CODE_OF_CONDUCT.md -tail -n +2 CODE_OF_CONDUCT.md >> ${OUTPUT_CONTENT_DIR}/CODE_OF_CONDUCT.md +cat CODE_OF_CONDUCT.md >> ${OUTPUT_CONTENT_DIR}/CODE_OF_CONDUCT.md echo "$(cat < ${OUTPUT_CONTENT_DIR}/CONTRIBUTING.md -tail -n +2 CONTRIBUTING.md >> ${OUTPUT_CONTENT_DIR}/CONTRIBUTING.md +cat CONTRIBUTING.md >> ${OUTPUT_CONTENT_DIR}/CONTRIBUTING.md echo "$(cat < ${OUTPUT_CONTENT_DIR}/CHANGELOG.md -tail -n +2 CHANGELOG.md >> ${OUTPUT_CONTENT_DIR}/CHANGELOG.md +cat CHANGELOG.md >> ${OUTPUT_CONTENT_DIR}/CHANGELOG.md echo "$(cat < ${OUTPUT_CONTENT_DIR}/MAINTAINERS.md -tail -n +2 MAINTAINERS.md >> ${OUTPUT_CONTENT_DIR}/MAINTAINERS.md +cat MAINTAINERS.md >> ${OUTPUT_CONTENT_DIR}/MAINTAINERS.md + +echo "$(cat < ${OUTPUT_CONTENT_DIR}/SECURITY.md +cat SECURITY.md >> ${OUTPUT_CONTENT_DIR}/SECURITY.md # Glob again to include new docs. ALL_DOC_CONTENT_FILES=$(echo "${OUTPUT_CONTENT_DIR}/**/*.md ${OUTPUT_CONTENT_DIR}/*.md") diff --git a/website/layouts/_default/baseof.html b/website/layouts/_default/baseof.html index d89163c3cc..e053dc6f6b 100644 --- a/website/layouts/_default/baseof.html +++ b/website/layouts/_default/baseof.html @@ -64,6 +64,7 @@
    Documentation
  • Storage
  • Service Discovery
  • Maintainers
  • +
  • Contributing
  • From f0a00ce53cd9edbc5394ef0ae11eba9102ece8a5 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 28 Jan 2020 08:34:04 +0000 Subject: [PATCH 204/257] Added security headers to our website. (#2056) This is required for GOLD CII Best practice badge: https://bestpractices.coreinfrastructure.org/en/projects/3048 Signed-off-by: Bartlomiej Plotka --- netlify.toml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/netlify.toml b/netlify.toml index f5c7576308..65b4238deb 100644 --- a/netlify.toml +++ b/netlify.toml @@ -16,4 +16,22 @@ command = "(env && make web HUGO=$(which hugo)) || (sleep 30; false)" [context.deploy-preview] # NOTE: Sleep at then is to make sure logs are not truncated on error. -command = "(env && make web HUGO=$(which hugo) WEBSITE_BASE_URL=${DEPLOY_PRIME_URL}) || (sleep 30; false)" \ No newline at end of file +command = "(env && make web HUGO=$(which hugo) WEBSITE_BASE_URL=${DEPLOY_PRIME_URL}) || (sleep 30; false)" + +[[headers]] + for = "/*" + [headers.values] + # We don't use iframes. Block them. + X-Frame-Options = "DENY" + # Don'a allow Mime-sniffing. + X-Content-Type-Options = "nosniff" + # Add reflective XSS protection. + X-XSS-Protection = "1; mode=block" + # Force HTTPS only. + Strict-Transport-Security = "max-age=31536000; includeSubDomains" + # Load scripts only via https and from white listed domains. + Content-Security-Policy = "default-src https:; script-src thanos.io" + # Only send referred when HTTPS is used. + Referrer-Policy = "strict-origin-when-cross-origin" + # Disable certain magic feature, lol. + Feature-Policy = "vibrate none; usermedia *; sync-xhr self" \ No newline at end of file From 273d9e9c4ad58689eebf7d4ef3417f60332d993d Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 28 Jan 2020 10:12:57 +0000 Subject: [PATCH 205/257] Updated stale bot as agreed offline on #thanos-dev. (#2058) Signed-off-by: Bartlomiej Plotka --- .github/stale.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index 0b024c09ed..415abad464 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -10,11 +10,9 @@ daysUntilClose: 7 # Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) onlyLabels: [] -# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable -exemptLabels: - - "priority: P0" - - bug - - pinned +# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable. +# We want stale bot to notify us that something is stale so we can revisit it. +exemptLabels: [] # Set to true to ignore issues in a project (defaults to false) exemptProjects: false @@ -30,8 +28,8 @@ staleLabel: stale # Comment to post when marking as stale. Set to `false` to disable markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you + This issue/PR is has been automatically marked as stale because it has not had + recent activity. Please comment on status otherwise the issue will be closed in a week. Thank you for your contributions. # Comment to post when removing the stale label. From 1f484661d320469df678913a05400bdf70ddb725 Mon Sep 17 00:00:00 2001 From: Kevin Hellemun <17928966+OGKevin@users.noreply.github.com> Date: Tue, 28 Jan 2020 11:14:13 +0100 Subject: [PATCH 206/257] Pass prom registry to shipper from ruler. (#2051) * Pass prom registry to shipper from ruler. https://github.com/thanos-io/thanos/issues/2050 Fixes #2050 Signed-off-by: Kevin Hellemun <17928966+OGKevin@users.noreply.github.com> * Updated CHANGELOG. https://github.com/thanos-io/thanos/pull/2050 Signed-off-by: Kevin Hellemun <17928966+OGKevin@users.noreply.github.com> --- CHANGELOG.md | 1 + cmd/thanos/rule.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81388ef318..f436611ff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#2033](https://github.com/thanos-io/thanos/pull/2033) minio-go: Fixed Issue #1494 support Web Identity providers for IAM credentials for AWS EKS - [#1985](https://github.com/thanos-io/thanos/pull/1985) store gateway: Fixed case where series entry is larger than 64KB in index. +- [#2051](https://github.com/thanos-io/thanos/pull/2051) ruler: Fixed issue where ruler does not expose shipper metrics. ### Added diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 5fcd9c2faa..76aef20317 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -629,7 +629,7 @@ func runRule( } }() - s := shipper.New(logger, nil, dataDir, bkt, func() labels.Labels { return lset }, metadata.RulerSource) + s := shipper.New(logger, reg, dataDir, bkt, func() labels.Labels { return lset }, metadata.RulerSource) ctx, cancel := context.WithCancel(context.Background()) From e743afed0d0876cdda2471271e64b5221926006d Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Tue, 28 Jan 2020 19:51:28 +0800 Subject: [PATCH 207/257] .github/stale.yml: fix a typo (#2059) Signed-off-by: Xiang Dai <764524258@qq.com> --- .github/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/stale.yml b/.github/stale.yml index 415abad464..ab344e9349 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -28,7 +28,7 @@ staleLabel: stale # Comment to post when marking as stale. Set to `false` to disable markComment: > - This issue/PR is has been automatically marked as stale because it has not had + This issue/PR has been automatically marked as stale because it has not had recent activity. Please comment on status otherwise the issue will be closed in a week. Thank you for your contributions. From f2a1702e20baf733fb9e89786856f3cf464b88d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Tue, 28 Jan 2020 13:02:36 +0100 Subject: [PATCH 208/257] SECURITY.md: clean up doc (#2061) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit cleans up the grammar in the SECURITY.md file for better readability. Signed-off-by: Lucas Servén Marín --- SECURITY.md | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 70cbeb48ad..9b60f2292d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,27 +1,31 @@ # Security Policy -As the Thanos team we are not security experts. However we try our best to avoid security concerns or to avoid -writing features that handles sensitive information at all. +As the Thanos team we are not security experts. +However we try our best to avoid security concerns and to avoid writing features that handle sensitive information at all. -It's worth to note that we assume the metric data to be sensitive and important. -External labels and query API parameters are treated as less sensitive, as they are logged and put into metric/traces. +It's worth noting that we assume metric data to be sensitive and important. +External labels and query API parameters are considered less sensitive, as they are logged and put into metric/traces. -## What you CAN expect: +## What You CAN Expect: -* We follow best programming practices. We test heavily including e2e tests against major object storages. We use vetting -and static analysis tool on every PR. We use secure protocols for building process (e.g to produce docker images) -* We don't log or put into our instrumentation any data that is stored in TSDB block. -* If we use crypto tools we always rely on FLOSS and standard libraries like official [Go crypt](https://golang.org/pkg/crypto/) +* We follow best programming practices. +We test heavily, including e2e tests against major object storages. +We use vetting and static analysis tools on every pull request. +We use secure protocols for building process, e.g. when producing Docker images. +* We don't put any data that is stored in TSDB into logs or instrumentation . +* If we use crypto tools, we always rely on FLOSS and standard libraries, like the official [Go crypt](https://golang.org/pkg/crypto/) library. * We always use TLS by default for communication with all object storages. -* We use stable Go versions to build our images and binaries. We update Go version as soon as new one is released. +* We use stable Go versions to build our images and binaries. +We update Go as soon as a new version is released. * We use only FLOSS tools. -## What we DON'T do (yet): +## What We DON'T Do (yet): -* We don't encrypt metric on local storage (e.g on disk). We don't do client encryption for object storage. We recommend -setting server side encryption for object storage. -* We don't allow to specify authorization or TLS for Thanos server HTTP APIs. +* We don't encrypt metrics in local storage, i.e. on disk. +We don't do client-side encryption for object storage. +We recommend setting server-side encryption for object storage. +* We don't allow specifying authorization or TLS for Thanos server HTTP APIs. ## Supported Versions @@ -32,4 +36,4 @@ setting server side encryption for object storage. ## Reporting a Vulnerability -If you encounter security vulnerability, please let us know privately via Thanos Team email: thanos-io@googlegroups.com +If you encounter a security vulnerability, please let us know privately via the Thanos Team email address: thanos-io@googlegroups.com. From b36b4b9fc0b6efed23b9432d4d9e5ab8b41bc270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Tue, 28 Jan 2020 13:03:50 +0100 Subject: [PATCH 209/257] netlify.toml: clean up comments (#2060) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit cleans up some comments that were recently added to the netlify.toml file. Signed-off-by: Lucas Servén Marín --- netlify.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/netlify.toml b/netlify.toml index 65b4238deb..20f4d31dfe 100644 --- a/netlify.toml +++ b/netlify.toml @@ -6,16 +6,16 @@ base = "" publish = "website/public" -# Our Makefile build hugo, but it's faster if netlify grab the correct version on their own -# via simple curl. +# Our Makefile builds Hugo, but it's faster if Netlify grabs the correct version on their own +# via a simple cURL. environment = { HUGO_VERSION="0.55.3" } -# NOTE: Sleep at then is to make sure logs are not truncated on error. +# NOTE: the sleep at the end is to make sure logs are not truncated on error. command = "(env && make web HUGO=$(which hugo)) || (sleep 30; false)" [context.deploy-preview] -# NOTE: Sleep at then is to make sure logs are not truncated on error. +# NOTE: the sleep at the end is to make sure logs are not truncated on error. command = "(env && make web HUGO=$(which hugo) WEBSITE_BASE_URL=${DEPLOY_PRIME_URL}) || (sleep 30; false)" [[headers]] @@ -23,15 +23,15 @@ command = "(env && make web HUGO=$(which hugo) WEBSITE_BASE_URL=${DEPLOY_PRIME_U [headers.values] # We don't use iframes. Block them. X-Frame-Options = "DENY" - # Don'a allow Mime-sniffing. + # Don't allow Mime-sniffing. X-Content-Type-Options = "nosniff" # Add reflective XSS protection. X-XSS-Protection = "1; mode=block" # Force HTTPS only. Strict-Transport-Security = "max-age=31536000; includeSubDomains" - # Load scripts only via https and from white listed domains. + # Load scripts only via HTTPS and from allowed domains. Content-Security-Policy = "default-src https:; script-src thanos.io" # Only send referred when HTTPS is used. Referrer-Policy = "strict-origin-when-cross-origin" - # Disable certain magic feature, lol. - Feature-Policy = "vibrate none; usermedia *; sync-xhr self" \ No newline at end of file + # Disable certain magic features, lol. + Feature-Policy = "vibrate none; usermedia *; sync-xhr self" From c7869d91d4a0967ca1bd3a7fbdbb206bb7379f3b Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Tue, 28 Jan 2020 15:28:09 +0100 Subject: [PATCH 210/257] querier, receiver, sidecar, store: Add gRPC health check endpoints (#2008) * Add gRPC health endpoints Signed-off-by: Kemal Akkoyun * Add grpc life-cycle methods Signed-off-by: Kemal Akkoyun * Utilize grpc health check server Signed-off-by: Kemal Akkoyun * Update CHANGELOG Signed-off-by: Kemal Akkoyun * Provide more structured way to probe Signed-off-by: Kemal Akkoyun * Fix linted errors Signed-off-by: Kemal Akkoyun * Remove white noise Signed-off-by: Kemal Akkoyun * Add dedicated probe for instrumentation Signed-off-by: Kemal Akkoyun * Retest Signed-off-by: Kemal Akkoyun --- CHANGELOG.md | 3 +- cmd/thanos/bucket.go | 11 +- cmd/thanos/compact.go | 10 +- cmd/thanos/downsample.go | 11 +- cmd/thanos/query.go | 16 ++- cmd/thanos/receive.go | 14 ++- cmd/thanos/rule.go | 15 ++- cmd/thanos/sidecar.go | 14 ++- cmd/thanos/store.go | 14 ++- pkg/prober/combiner.go | 53 ++++++++ pkg/prober/grpc.go | 44 +++++++ pkg/prober/http.go | 79 ++++++++++++ pkg/prober/{prober_test.go => http_test.go} | 50 ++++---- pkg/prober/intrumentation.go | 68 +++++++++++ pkg/prober/prober.go | 127 +------------------- pkg/server/grpc/grpc.go | 10 +- pkg/server/http/http.go | 12 +- 17 files changed, 360 insertions(+), 191 deletions(-) create mode 100644 pkg/prober/combiner.go create mode 100644 pkg/prober/grpc.go create mode 100644 pkg/prober/http.go rename pkg/prober/{prober_test.go => http_test.go} (77%) create mode 100644 pkg/prober/intrumentation.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f436611ff4..dee286e74c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,8 +24,9 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`. - [#1939](https://github.com/thanos-io/thanos/pull/1939) Ruler: Add TLS and authentication support for query endpoints with the `--query.config` and `--query.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. - [#1982](https://github.com/thanos-io/thanos/pull/1982) Ruler: Add support for Alertmanager v2 API endpoints. -- #2030 Query: Add `thanos_proxy_store_empty_stream_responses_total` metric for number of empty responses from stores. +- [#2030](https://github.com/thanos-io/thanos/pull/2030) Query: Add `thanos_proxy_store_empty_stream_responses_total` metric for number of empty responses from stores. - [#2049](https://github.com/thanos-io/thanos/pull/2049) Tracing: Support sampling on Elastic APM with new sample_rate setting. +- [#2008](https://github.com/thanos-io/thanos/pull/2008) Querier, Receiver, Sidecar, Store: Add gRPC [health check](https://github.com/grpc/grpc/blob/master/doc/health-checking.md) endpoints. ### Changed diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index d49c360659..b7bb9ea584 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -316,9 +316,14 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str m[name+" web"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { ctx, cancel := context.WithCancel(context.Background()) - statusProber := prober.New(component.Bucket, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) - // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := httpserver.New(logger, reg, component.Bucket, statusProber, + comp := component.Bucket + httpProbe := prober.NewHTTP() + statusProber := prober.Combine( + httpProbe, + prober.NewInstrumentation(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)), + ) + + srv := httpserver.New(logger, reg, comp, httpProbe, httpserver.WithListen(*httpBindAddr), httpserver.WithGracePeriod(time.Duration(*httpGracePeriod)), ) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 5832625158..084630971a 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -195,9 +195,13 @@ func runCompact( downsampleMetrics := newDownsampleMetrics(reg) - statusProber := prober.New(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) - // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := httpserver.New(logger, reg, component, statusProber, + httpProbe := prober.NewHTTP() + statusProber := prober.Combine( + httpProbe, + prober.NewInstrumentation(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)), + ) + + srv := httpserver.New(logger, reg, component, httpProbe, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 00de9e57cb..bf5cf7e58f 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -101,8 +101,13 @@ func runDownsample( } }() + httpProbe := prober.NewHTTP() + statusProber := prober.Combine( + httpProbe, + prober.NewInstrumentation(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)), + ) + metrics := newDownsampleMetrics(reg) - statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Start cycle of syncing blocks from the bucket and garbage collecting the bucket. { ctx, cancel := context.WithCancel(context.Background()) @@ -129,11 +134,11 @@ func runDownsample( }) } - // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := httpserver.New(logger, reg, comp, statusProber, + srv := httpserver.New(logger, reg, comp, httpProbe, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) + g.Add(func() error { statusProber.Healthy() diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 9d3412db31..159a3ef16e 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -308,9 +308,16 @@ func runQuery( cancel() }) } - // Start query API + UI HTTP server. - statusProber := prober.New(comp, logger, reg) + grpcProbe := prober.NewGRPC() + httpProbe := prober.NewHTTP() + statusProber := prober.Combine( + httpProbe, + grpcProbe, + prober.NewInstrumentation(comp, logger, reg), + ) + + // Start query API + UI HTTP server. { router := route.New() @@ -334,8 +341,7 @@ func runQuery( api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins) - // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := httpserver.New(logger, reg, comp, statusProber, + srv := httpserver.New(logger, reg, comp, httpProbe, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) @@ -359,7 +365,7 @@ func runQuery( return errors.Wrap(err, "setup gRPC server") } - s := grpcserver.New(logger, reg, tracer, comp, proxy, + s := grpcserver.New(logger, reg, tracer, comp, grpcProbe, proxy, grpcserver.WithListen(grpcBindAddr), grpcserver.WithGracePeriod(grpcGracePeriod), grpcserver.WithTLSConfig(tlsCfg), diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index ba552e67e3..71b2194c91 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -209,7 +209,14 @@ func runReceive( DialOpts: dialOpts, }) - statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + grpcProbe := prober.NewGRPC() + httpProbe := prober.NewHTTP() + statusProber := prober.Combine( + httpProbe, + grpcProbe, + prober.NewInstrumentation(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)), + ) + confContentYaml, err := objStoreConfig.Content() if err != nil { return err @@ -351,8 +358,7 @@ func runReceive( } level.Debug(logger).Log("msg", "setting up http server") - // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := httpserver.New(logger, reg, comp, statusProber, + srv := httpserver.New(logger, reg, comp, httpProbe, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) @@ -389,7 +395,7 @@ func runReceive( WriteableStoreServer: webHandler, } - s = grpcserver.NewReadWrite(logger, &receive.UnRegisterer{Registerer: reg}, tracer, comp, rw, + s = grpcserver.NewReadWrite(logger, &receive.UnRegisterer{Registerer: reg}, tracer, comp, grpcProbe, rw, grpcserver.WithListen(grpcBindAddr), grpcserver.WithGracePeriod(grpcGracePeriod), grpcserver.WithTLSConfig(tlsCfg), diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 76aef20317..db7edd7144 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -538,7 +538,15 @@ func runRule( close(cancel) }) } - statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) + + grpcProbe := prober.NewGRPC() + httpProbe := prober.NewHTTP() + statusProber := prober.Combine( + httpProbe, + grpcProbe, + prober.NewInstrumentation(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)), + ) + // Start gRPC server. { store := store.NewTSDBStore(logger, reg, db, component.Rule, lset) @@ -548,7 +556,7 @@ func runRule( return errors.Wrap(err, "setup gRPC server") } - s := grpcserver.New(logger, reg, tracer, comp, store, + s := grpcserver.New(logger, reg, tracer, comp, grpcProbe, store, grpcserver.WithListen(grpcBindAddr), grpcserver.WithGracePeriod(grpcGracePeriod), grpcserver.WithTLSConfig(tlsCfg), @@ -590,8 +598,7 @@ func runRule( api := v1.NewAPI(logger, reg, ruleMgr) api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins) - // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - srv := httpserver.New(logger, reg, comp, statusProber, + srv := httpserver.New(logger, reg, comp, httpProbe, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index bd662e87e3..0b7babf2d1 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -154,9 +154,15 @@ func runSidecar( uploads = false } - // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - statusProber := prober.New(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) - srv := httpserver.New(logger, reg, comp, statusProber, + grpcProbe := prober.NewGRPC() + httpProbe := prober.NewHTTP() + statusProber := prober.Combine( + httpProbe, + grpcProbe, + prober.NewInstrumentation(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)), + ) + + srv := httpserver.New(logger, reg, comp, httpProbe, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) @@ -269,7 +275,7 @@ func runSidecar( return errors.Wrap(err, "setup gRPC server") } - s := grpcserver.New(logger, reg, tracer, comp, promStore, + s := grpcserver.New(logger, reg, tracer, comp, grpcProbe, promStore, grpcserver.WithListen(grpcBindAddr), grpcserver.WithGracePeriod(grpcGracePeriod), grpcserver.WithTLSConfig(tlsCfg), diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 5e19037be1..dc3a0cf225 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -146,9 +146,15 @@ func runStore( advertiseCompatibilityLabel bool, enableIndexHeader bool, ) error { - // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. - statusProber := prober.New(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) - srv := httpserver.New(logger, reg, component, statusProber, + grpcProbe := prober.NewGRPC() + httpProbe := prober.NewHTTP() + statusProber := prober.Combine( + httpProbe, + grpcProbe, + prober.NewInstrumentation(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)), + ) + + srv := httpserver.New(logger, reg, component, httpProbe, httpserver.WithListen(httpBindAddr), httpserver.WithGracePeriod(httpGracePeriod), ) @@ -278,7 +284,7 @@ func runStore( return errors.Wrap(err, "setup gRPC server") } - s := grpcserver.New(logger, reg, tracer, component, bs, + s := grpcserver.New(logger, reg, tracer, component, grpcProbe, bs, grpcserver.WithListen(grpcBindAddr), grpcserver.WithGracePeriod(grpcGracePeriod), grpcserver.WithTLSConfig(tlsCfg), diff --git a/pkg/prober/combiner.go b/pkg/prober/combiner.go new file mode 100644 index 0000000000..b85d6a195e --- /dev/null +++ b/pkg/prober/combiner.go @@ -0,0 +1,53 @@ +package prober + +import "sync" + +type combined struct { + mu sync.Mutex + probes []Probe +} + +// Combine folds given probes into one, reflects their statuses in a thread-safe way. +func Combine(probes ...Probe) Probe { + return &combined{probes: probes} +} + +// Ready sets components status to ready. +func (p *combined) Ready() { + p.mu.Lock() + defer p.mu.Unlock() + + for _, probe := range p.probes { + probe.Ready() + } +} + +// NotReady sets components status to not ready with given error as a cause. +func (p *combined) NotReady(err error) { + p.mu.Lock() + defer p.mu.Unlock() + + for _, probe := range p.probes { + probe.NotReady(err) + } +} + +// Healthy sets components status to healthy. +func (p *combined) Healthy() { + p.mu.Lock() + defer p.mu.Unlock() + + for _, probe := range p.probes { + probe.Healthy() + } +} + +// NotHealthy sets components status to not healthy with given error as a cause. +func (p *combined) NotHealthy(err error) { + p.mu.Lock() + defer p.mu.Unlock() + + for _, probe := range p.probes { + probe.NotHealthy(err) + } +} diff --git a/pkg/prober/grpc.go b/pkg/prober/grpc.go new file mode 100644 index 0000000000..117ced0427 --- /dev/null +++ b/pkg/prober/grpc.go @@ -0,0 +1,44 @@ +package prober + +import ( + "google.golang.org/grpc/health" + grpc_health "google.golang.org/grpc/health/grpc_health_v1" +) + +// GRPCProbe represents health and readiness status of given component, and provides GRPC integration. +type GRPCProbe struct { + h *health.Server +} + +// NewGRPC creates a Probe that wrapped around grpc/healt.Server which reflects status of server. +func NewGRPC() *GRPCProbe { + h := health.NewServer() + h.SetServingStatus("", grpc_health.HealthCheckResponse_NOT_SERVING) + + return &GRPCProbe{h: h} +} + +// HealthServer returns a gRPC health server which responds readiness and liveness checks. +func (p *GRPCProbe) HealthServer() *health.Server { + return p.h +} + +// Ready sets components status to ready. +func (p *GRPCProbe) Ready() { + p.h.SetServingStatus("", grpc_health.HealthCheckResponse_SERVING) +} + +// NotReady sets components status to not ready with given error as a cause. +func (p *GRPCProbe) NotReady(err error) { + p.h.SetServingStatus("", grpc_health.HealthCheckResponse_NOT_SERVING) +} + +// Healthy sets components status to healthy. +func (p *GRPCProbe) Healthy() { + p.h.Resume() +} + +// NotHealthy sets components status to not healthy with given error as a cause. +func (p *GRPCProbe) NotHealthy(err error) { + p.h.Shutdown() +} diff --git a/pkg/prober/http.go b/pkg/prober/http.go new file mode 100644 index 0000000000..96777a09b2 --- /dev/null +++ b/pkg/prober/http.go @@ -0,0 +1,79 @@ +package prober + +import ( + "io" + "net/http" + "sync/atomic" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" +) + +type check func() bool + +// HTTPProbe represents health and readiness status of given component, and provides HTTP integration. +type HTTPProbe struct { + ready uint32 + healthy uint32 +} + +// NewHTTP returns HTTPProbe representing readiness and healthiness of given component. +func NewHTTP() *HTTPProbe { + return &HTTPProbe{} +} + +// HealthyHandler returns a HTTP Handler which responds health checks. +func (p *HTTPProbe) HealthyHandler(logger log.Logger) http.HandlerFunc { + return p.handler(logger, p.isHealthy) +} + +// ReadyHandler returns a HTTP Handler which responds readiness checks. +func (p *HTTPProbe) ReadyHandler(logger log.Logger) http.HandlerFunc { + return p.handler(logger, p.isReady) +} + +func (p *HTTPProbe) handler(logger log.Logger, c check) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + if !c() { + http.Error(w, "NOT OK", http.StatusServiceUnavailable) + return + } + if _, err := io.WriteString(w, "OK"); err != nil { + level.Error(logger).Log("msg", "failed to write probe response", "err", err) + } + } +} + +// isReady returns true if component is ready. +func (p *HTTPProbe) isReady() bool { + ready := atomic.LoadUint32(&p.ready) + return ready > 0 +} + +// isHealthy returns true if component is healthy. +func (p *HTTPProbe) isHealthy() bool { + healthy := atomic.LoadUint32(&p.healthy) + return healthy > 0 +} + +// Ready sets components status to ready. +func (p *HTTPProbe) Ready() { + atomic.SwapUint32(&p.ready, 1) +} + +// NotReady sets components status to not ready with given error as a cause. +func (p *HTTPProbe) NotReady(err error) { + atomic.SwapUint32(&p.ready, 0) + +} + +// Healthy sets components status to healthy. +func (p *HTTPProbe) Healthy() { + atomic.SwapUint32(&p.healthy, 1) + +} + +// NotHealthy sets components status to not healthy with given error as a cause. +func (p *HTTPProbe) NotHealthy(err error) { + atomic.SwapUint32(&p.healthy, 0) +} diff --git a/pkg/prober/prober_test.go b/pkg/prober/http_test.go similarity index 77% rename from pkg/prober/prober_test.go rename to pkg/prober/http_test.go index 694bc105d2..955cbd3921 100644 --- a/pkg/prober/prober_test.go +++ b/pkg/prober/http_test.go @@ -13,38 +13,21 @@ import ( "github.com/thanos-io/thanos/pkg/testutil" ) -func doGet(ctx context.Context, url string) (*http.Response, error) { - req, err := http.NewRequest("GET", fmt.Sprintf("http://%s", url), nil) - if err != nil { - return nil, err - } - - return http.DefaultClient.Do(req.WithContext(ctx)) -} - -type TestComponent struct { - name string -} - -func (c TestComponent) String() string { - return c.name -} - -func TestProberHealthInitialState(t *testing.T) { - p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) +func TestHTTPProberHealthInitialState(t *testing.T) { + p := NewHTTP() testutil.Assert(t, !p.isHealthy(), "initially should not be healthy") } -func TestProberReadinessInitialState(t *testing.T) { - p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) +func TestHTTPProberReadinessInitialState(t *testing.T) { + p := NewHTTP() testutil.Assert(t, !p.isReady(), "initially should not be ready") } -func TestProberHealthyStatusSetting(t *testing.T) { +func TestHTTPProberHealthyStatusSetting(t *testing.T) { testError := fmt.Errorf("test error") - p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) + p := NewHTTP() p.Healthy() @@ -55,9 +38,9 @@ func TestProberHealthyStatusSetting(t *testing.T) { testutil.Assert(t, !p.isHealthy(), "should not be healthy") } -func TestProberReadyStatusSetting(t *testing.T) { +func TestHTTPProberReadyStatusSetting(t *testing.T) { testError := fmt.Errorf("test error") - p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) + p := NewHTTP() p.Ready() @@ -68,20 +51,20 @@ func TestProberReadyStatusSetting(t *testing.T) { testutil.Assert(t, !p.isReady(), "should not be ready") } -func TestProberMuxRegistering(t *testing.T) { +func TestHTTPProberMuxRegistering(t *testing.T) { serverAddress := fmt.Sprintf("localhost:%d", 8081) l, err := net.Listen("tcp", serverAddress) testutil.Ok(t, err) - p := New(TestComponent{name: "test"}, log.NewNopLogger(), nil) + p := NewHTTP() healthyEndpointPath := "/-/healthy" readyEndpointPath := "/-/ready" mux := http.NewServeMux() - mux.HandleFunc(healthyEndpointPath, p.HealthyHandler()) - mux.HandleFunc(readyEndpointPath, p.ReadyHandler()) + mux.HandleFunc(healthyEndpointPath, p.HealthyHandler(log.NewNopLogger())) + mux.HandleFunc(readyEndpointPath, p.ReadyHandler(log.NewNopLogger())) var g run.Group g.Add(func() error { @@ -137,3 +120,12 @@ func TestProberMuxRegistering(t *testing.T) { testutil.Equals(t, resp.StatusCode, http.StatusOK, "should be ready, response code: %d", resp.StatusCode) } } + +func doGet(ctx context.Context, url string) (*http.Response, error) { + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s", url), nil) + if err != nil { + return nil, err + } + + return http.DefaultClient.Do(req.WithContext(ctx)) +} diff --git a/pkg/prober/intrumentation.go b/pkg/prober/intrumentation.go new file mode 100644 index 0000000000..8a9751de93 --- /dev/null +++ b/pkg/prober/intrumentation.go @@ -0,0 +1,68 @@ +package prober + +import ( + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" + + "github.com/thanos-io/thanos/pkg/component" +) + +const ( + ready = "ready" + healthy = "healthy" +) + +// InstrumentationProbe stores instrumentation state of Probe. +// This is created with an intention to combine with other Probe's using prober.Combine. +type InstrumentationProbe struct { + component component.Component + logger log.Logger + + status *prometheus.GaugeVec +} + +// NewInstrumentation returns InstrumentationProbe records readiness and healthiness for given component. +func NewInstrumentation(component component.Component, logger log.Logger, reg prometheus.Registerer) *InstrumentationProbe { + p := InstrumentationProbe{ + component: component, + logger: logger, + status: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "status", + Help: "Represents status (0 indicates success, 1 indicates failure) of the component.", + ConstLabels: map[string]string{"component": component.String()}, + }, + []string{"check"}, + ), + } + + if reg != nil { + reg.MustRegister(p.status) + } + + return &p +} + +// Ready records the component status when Ready is called, if combined with other Probes. +func (p *InstrumentationProbe) Ready() { + p.status.WithLabelValues(ready).Set(1) + level.Info(p.logger).Log("msg", "changing probe status", "status", "ready") +} + +// NotReady records the component status when NotReady is called, if combined with other Probes. +func (p *InstrumentationProbe) NotReady(err error) { + p.status.WithLabelValues(ready).Set(0) + level.Warn(p.logger).Log("msg", "changing probe status", "status", "not-ready", "reason", err) +} + +// Healthy records the component status when Healthy is called, if combined with other Probes. +func (p *InstrumentationProbe) Healthy() { + p.status.WithLabelValues(healthy).Set(1) + level.Info(p.logger).Log("msg", "changing probe status", "status", "healthy") +} + +// NotHealthy records the component status when NotHealthy is called, if combined with other Probes. +func (p *InstrumentationProbe) NotHealthy(err error) { + p.status.WithLabelValues(healthy).Set(0) + level.Info(p.logger).Log("msg", "changing probe status", "status", "not-healthy", "reason", err) +} diff --git a/pkg/prober/prober.go b/pkg/prober/prober.go index c1d38fbd11..87bff192e3 100644 --- a/pkg/prober/prober.go +++ b/pkg/prober/prober.go @@ -1,24 +1,5 @@ package prober -import ( - "io" - "net/http" - "sync/atomic" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/thanos-io/thanos/pkg/component" -) - -type check func() bool - -const ( - ready = "ready" - healthy = "healthy" -) - // Prober represents health and readiness status of given component. // // From Kubernetes documentation https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ : @@ -33,107 +14,9 @@ const ( // but you don’t want to send it requests either. Kubernetes provides readiness probes to detect // and mitigate these situations. A pod with containers reporting that they are not ready // does not receive traffic through Kubernetes Services. -type Prober struct { - component component.Component - logger log.Logger - - ready uint32 - healthy uint32 - - status *prometheus.GaugeVec -} - -// New returns Prober representing readiness and healthiness of given component. -func New(component component.Component, logger log.Logger, reg prometheus.Registerer) *Prober { - p := &Prober{ - component: component, - logger: logger, - status: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "status", - Help: "Represents status (0 indicates success, 1 indicates failure) of the component.", - ConstLabels: map[string]string{"component": component.String()}, - }, - []string{"check"}, - ), - } - - if reg != nil { - reg.MustRegister(p.status) - } - - return p -} - -// HealthyHandler returns a HTTP Handler which responds health checks. -func (p *Prober) HealthyHandler() http.HandlerFunc { - return p.handler(p.isHealthy) -} - -// ReadyHandler returns a HTTP Handler which responds readiness checks. -func (p *Prober) ReadyHandler() http.HandlerFunc { - return p.handler(p.isReady) -} - -func (p *Prober) handler(c check) http.HandlerFunc { - return func(w http.ResponseWriter, _ *http.Request) { - if !c() { - http.Error(w, "NOT OK", http.StatusServiceUnavailable) - return - } - if _, err := io.WriteString(w, "OK"); err != nil { - level.Error(p.logger).Log("msg", "failed to write probe response", "err", err) - } - } -} - -// isReady returns true if component is ready. -func (p *Prober) isReady() bool { - ready := atomic.LoadUint32(&p.ready) - return ready > 0 -} - -// isHealthy returns true if component is healthy. -func (p *Prober) isHealthy() bool { - healthy := atomic.LoadUint32(&p.healthy) - return healthy > 0 -} - -// Ready sets components status to ready. -func (p *Prober) Ready() { - old := atomic.SwapUint32(&p.ready, 1) - - if old == 0 { - p.status.WithLabelValues(ready).Set(1) - level.Info(p.logger).Log("msg", "changing probe status", "status", "ready") - } -} - -// NotReady sets components status to not ready with given error as a cause. -func (p *Prober) NotReady(err error) { - old := atomic.SwapUint32(&p.ready, 0) - - if old == 1 { - p.status.WithLabelValues(ready).Set(0) - level.Warn(p.logger).Log("msg", "changing probe status", "status", "not-ready", "reason", err) - } -} - -// Healthy sets components status to healthy. -func (p *Prober) Healthy() { - old := atomic.SwapUint32(&p.healthy, 1) - - if old == 0 { - p.status.WithLabelValues(healthy).Set(1) - level.Info(p.logger).Log("msg", "changing probe status", "status", "healthy") - } -} - -// NotHealthy sets components status to not healthy with given error as a cause. -func (p *Prober) NotHealthy(err error) { - old := atomic.SwapUint32(&p.healthy, 0) - - if old == 1 { - p.status.WithLabelValues(healthy).Set(0) - level.Info(p.logger).Log("msg", "changing probe status", "status", "not-healthy", "reason", err) - } +type Probe interface { + Healthy() + NotHealthy(err error) + Ready() + NotReady(err error) } diff --git a/pkg/server/grpc/grpc.go b/pkg/server/grpc/grpc.go index 38b1a1a3ac..b1b7779541 100644 --- a/pkg/server/grpc/grpc.go +++ b/pkg/server/grpc/grpc.go @@ -15,11 +15,13 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/tracing" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + grpc_health "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/status" ) @@ -35,7 +37,7 @@ type Server struct { } // New creates a new Server. -func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, comp component.Component, storeSrv storepb.StoreServer, opts ...Option) *Server { +func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, comp component.Component, probe *prober.GRPCProbe, storeSrv storepb.StoreServer, opts ...Option) *Server { options := options{} for _, o := range opts { o.apply(&options) @@ -79,6 +81,8 @@ func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer storepb.RegisterStoreServer(s, storeSrv) met.InitializeMetrics(s) + grpc_health.RegisterHealthServer(s, probe.HealthServer()) + return &Server{ logger: log.With(logger, "service", "gRPC/server", "component", comp.String()), comp: comp, @@ -135,8 +139,8 @@ type ReadWriteStoreServer interface { } // NewReadWrite creates a new server that can be written to. -func NewReadWrite(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, comp component.Component, storeSrv ReadWriteStoreServer, opts ...Option) *Server { - s := New(logger, reg, tracer, comp, storeSrv, opts...) +func NewReadWrite(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, comp component.Component, probe *prober.GRPCProbe, storeSrv ReadWriteStoreServer, opts ...Option) *Server { + s := New(logger, reg, tracer, comp, probe, storeSrv, opts...) storepb.RegisterWriteableStoreServer(s.srv, storeSrv) return s } diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index 56e7e87268..52e839d9f5 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -18,7 +18,7 @@ import ( type Server struct { logger log.Logger comp component.Component - prober *prober.Prober + prober *prober.HTTPProbe mux *http.ServeMux srv *http.Server @@ -27,7 +27,7 @@ type Server struct { } // New creates a new Server. -func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, prober *prober.Prober, opts ...Option) *Server { +func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, prober *prober.HTTPProbe, opts ...Option) *Server { options := options{} for _, o := range opts { o.apply(&options) @@ -35,7 +35,7 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, mux := http.NewServeMux() registerMetrics(mux, reg) - registerProbes(mux, prober) + registerProbes(mux, prober, logger) registerProfiler(mux) return &Server{ @@ -96,9 +96,9 @@ func registerMetrics(mux *http.ServeMux, g prometheus.Gatherer) { } } -func registerProbes(mux *http.ServeMux, p *prober.Prober) { +func registerProbes(mux *http.ServeMux, p *prober.HTTPProbe, logger log.Logger) { if p != nil { - mux.Handle("/-/healthy", p.HealthyHandler()) - mux.Handle("/-/ready", p.ReadyHandler()) + mux.Handle("/-/healthy", p.HealthyHandler(logger)) + mux.Handle("/-/ready", p.ReadyHandler(logger)) } } From d4279663fab7682a029b783556a0793aec6b6a7b Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Tue, 28 Jan 2020 16:20:56 +0100 Subject: [PATCH 211/257] mixin, docs: Rename Compact -> Compactor (#2065) * Rename compact to compactor in mixin Signed-off-by: Kemal Akkoyun * Change name in docs Signed-off-by: Kemal Akkoyun * Fix compactor alert name Signed-off-by: Kemal Akkoyun --- docs/components/compact.md | 4 ++-- examples/alerts/alerts.md | 14 +++++++------- examples/alerts/alerts.yaml | 14 +++++++------- examples/dashboards/overview.json | 2 +- mixin/thanos/alerts/compactor.libsonnet | 16 ++++++++-------- mixin/thanos/dashboards/compactor.libsonnet | 8 ++++---- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/components/compact.md b/docs/components/compact.md index d3c3b78a55..c41b1ff2bc 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -1,10 +1,10 @@ --- -title: Compact +title: Compactor type: docs menu: components --- -# Compact +# Compactor The compactor component of Thanos applies the compaction procedure of the Prometheus 2.0 storage engine to block data stored in object storage. It is generally not semantically concurrency safe and must be deployed as a singleton against a bucket. diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index dad259a3d0..3548287a63 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -8,24 +8,24 @@ Here are some example alerts configured for Kubernetes environment. ```yaml name: thanos-compactor.rules rules: -- alert: ThanosCompactorMultipleCompactsAreRunning +- alert: ThanosCompactorMultipleCompactorsAreRunning annotations: - message: You should never run more than one Thanos Compact at once. You have {{ - $value }} + message: You should never run more than one Thanos Compactor at once. You have + {{ $value }} expr: sum(up{job=~"thanos-compactor.*"}) > 1 for: 5m labels: severity: warning - alert: ThanosCompactorHalted annotations: - message: Thanos Compact {{$labels.job}} has failed to run and now is halted. + message: Thanos Compactor {{$labels.job}} has failed to run and now is halted. expr: thanos_compactor_halted{job=~"thanos-compactor.*"} == 1 for: 5m labels: severity: warning - alert: ThanosCompactorHighCompactionFailures annotations: - message: Thanos Compact {{$labels.job}} is failing to execute {{ $value | humanize + message: Thanos Compactor {{$labels.job}} is failing to execute {{ $value | humanize }}% of compactions. expr: | ( @@ -39,7 +39,7 @@ rules: severity: warning - alert: ThanosCompactorBucketHighOperationFailures annotations: - message: Thanos Compact {{$labels.job}} Bucket is failing to execute {{ $value + message: Thanos Compactor {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations. expr: | ( @@ -53,7 +53,7 @@ rules: severity: warning - alert: ThanosCompactorHasNotRun annotations: - message: Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours. + message: Thanos Compactor {{$labels.job}} has not uploaded anything for 24 hours. expr: (time() - max(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compactor.*"})) / 60 / 60 > 24 labels: diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index 5011e882b4..73579f897a 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -1,9 +1,9 @@ groups: - name: thanos-compactor.rules rules: - - alert: ThanosCompactorMultipleCompactsAreRunning + - alert: ThanosCompactorMultipleCompactorsAreRunning annotations: - message: You should never run more than one Thanos Compact at once. You have + message: You should never run more than one Thanos Compactor at once. You have {{ $value }} expr: sum(up{job=~"thanos-compactor.*"}) > 1 for: 5m @@ -11,15 +11,15 @@ groups: severity: warning - alert: ThanosCompactorHalted annotations: - message: Thanos Compact {{$labels.job}} has failed to run and now is halted. + message: Thanos Compactor {{$labels.job}} has failed to run and now is halted. expr: thanos_compactor_halted{job=~"thanos-compactor.*"} == 1 for: 5m labels: severity: warning - alert: ThanosCompactorHighCompactionFailures annotations: - message: Thanos Compact {{$labels.job}} is failing to execute {{ $value | humanize - }}% of compactions. + message: Thanos Compactor {{$labels.job}} is failing to execute {{ $value | + humanize }}% of compactions. expr: | ( sum by (job) (rate(thanos_compact_group_compactions_failures_total{job=~"thanos-compactor.*"}[5m])) @@ -32,7 +32,7 @@ groups: severity: warning - alert: ThanosCompactorBucketHighOperationFailures annotations: - message: Thanos Compact {{$labels.job}} Bucket is failing to execute {{ $value + message: Thanos Compactor {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations. expr: | ( @@ -46,7 +46,7 @@ groups: severity: warning - alert: ThanosCompactorHasNotRun annotations: - message: Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours. + message: Thanos Compactor {{$labels.job}} has not uploaded anything for 24 hours. expr: (time() - max(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compactor.*"})) / 60 / 60 > 24 labels: diff --git a/examples/dashboards/overview.json b/examples/dashboards/overview.json index f328a3e370..ebd45038ca 100644 --- a/examples/dashboards/overview.json +++ b/examples/dashboards/overview.json @@ -1968,7 +1968,7 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Compact", + "title": "Compactor", "titleSize": "h6" } ], diff --git a/mixin/thanos/alerts/compactor.libsonnet b/mixin/thanos/alerts/compactor.libsonnet index 6c34259b9b..3baab8b7d3 100644 --- a/mixin/thanos/alerts/compactor.libsonnet +++ b/mixin/thanos/alerts/compactor.libsonnet @@ -1,8 +1,8 @@ { local thanos = self, compactor+:: { - jobPrefix: error 'must provide job prefix for Thanos Compact alerts', - selector: error 'must provide selector for Thanos Compact alerts', + jobPrefix: error 'must provide job prefix for Thanos Compactor alerts', + selector: error 'must provide selector for Thanos Compactor alerts', }, prometheusAlerts+:: { groups+: [ @@ -10,9 +10,9 @@ name: 'thanos-compactor.rules', rules: [ { - alert: 'ThanosCompactorMultipleCompactsAreRunning', + alert: 'ThanosCompactorMultipleCompactorsAreRunning', annotations: { - message: 'You should never run more than one Thanos Compact at once. You have {{ $value }}', + message: 'You should never run more than one Thanos Compactor at once. You have {{ $value }}', }, expr: 'sum(up{%(selector)s}) > 1' % thanos.compactor, 'for': '5m', @@ -23,7 +23,7 @@ { alert: 'ThanosCompactorHalted', annotations: { - message: 'Thanos Compact {{$labels.job}} has failed to run and now is halted.', + message: 'Thanos Compactor {{$labels.job}} has failed to run and now is halted.', }, expr: 'thanos_compactor_halted{%(selector)s} == 1' % thanos.compactor, 'for': '5m', @@ -34,7 +34,7 @@ { alert: 'ThanosCompactorHighCompactionFailures', annotations: { - message: 'Thanos Compact {{$labels.job}} is failing to execute {{ $value | humanize }}% of compactions.', + message: 'Thanos Compactor {{$labels.job}} is failing to execute {{ $value | humanize }}% of compactions.', }, expr: ||| ( @@ -52,7 +52,7 @@ { alert: 'ThanosCompactorBucketHighOperationFailures', annotations: { - message: 'Thanos Compact {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations.', + message: 'Thanos Compactor {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations.', }, expr: ||| ( @@ -70,7 +70,7 @@ { alert: 'ThanosCompactorHasNotRun', annotations: { - message: 'Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours.', + message: 'Thanos Compactor {{$labels.job}} has not uploaded anything for 24 hours.', }, expr: '(time() - max(thanos_objstore_bucket_last_successful_upload_time{%(selector)s})) / 60 / 60 > 24' % thanos.compactor, labels: { diff --git a/mixin/thanos/dashboards/compactor.libsonnet b/mixin/thanos/dashboards/compactor.libsonnet index af34618e04..35e9c18c61 100644 --- a/mixin/thanos/dashboards/compactor.libsonnet +++ b/mixin/thanos/dashboards/compactor.libsonnet @@ -3,9 +3,9 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, compactor+:: { - jobPrefix: error 'must provide job prefix for Thanos Compact dashboard', - selector: error 'must provide selector for Thanos Compact dashboard', - title: error 'must provide title for Thanos Compact dashboard', + jobPrefix: error 'must provide job prefix for Thanos Compactor dashboard', + selector: error 'must provide selector for Thanos Compactor dashboard', + title: error 'must provide title for Thanos Compactor dashboard', }, grafanaDashboards+:: { 'compactor.json': @@ -135,7 +135,7 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.compactor, true, '.*'), __overviewRows__+:: [ - g.row('Compact') + g.row('Compactor') .addPanel( g.panel( 'Compaction Rate', From 55cb8ca38b3539381dc6a781e637df15c694e50a Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 28 Jan 2020 16:25:36 +0000 Subject: [PATCH 212/257] Added Copyright header to all Go and Proto source files. (#2063) * Added Copyright header to all Go and Proto source files. As per: https://github.com/cncf/foundation/blob/master/copyright-notices.md#copyright-notices Signed-off-by: Bartlomiej Plotka * Added Copyright header to all Go and Proto source files. As per: https://github.com/cncf/foundation/blob/master/copyright-notices.md#copyright-notices Signed-off-by: Bartlomiej Plotka --- Makefile | 3 + cmd/thanos/bucket.go | 3 + cmd/thanos/check.go | 3 + cmd/thanos/check_test.go | 3 + cmd/thanos/compact.go | 3 + cmd/thanos/downsample.go | 3 + cmd/thanos/flags.go | 3 + cmd/thanos/main.go | 3 + cmd/thanos/main_test.go | 3 + cmd/thanos/query.go | 3 + cmd/thanos/receive.go | 3 + cmd/thanos/rule.go | 3 + cmd/thanos/rule_test.go | 3 + cmd/thanos/sidecar.go | 3 + cmd/thanos/store.go | 3 + doc.go | 3 + pkg/alert/alert.go | 3 + pkg/alert/alert_test.go | 3 + pkg/alert/config.go | 3 + pkg/alert/config_test.go | 3 + pkg/block/block.go | 3 + pkg/block/block_test.go | 3 + pkg/block/fetcher.go | 3 + pkg/block/fetcher_test.go | 3 + pkg/block/index.go | 3 + pkg/block/index_test.go | 3 + pkg/block/indexheader/binary_reader.go | 3 + pkg/block/indexheader/header.go | 3 + pkg/block/indexheader/header_test.go | 3 + pkg/block/indexheader/json_reader.go | 3 + pkg/block/metadata/meta.go | 3 + pkg/cacheutil/jump_hash.go | 3 + pkg/cacheutil/memcached_client.go | 3 + pkg/cacheutil/memcached_client_test.go | 3 + pkg/cacheutil/memcached_server_selector.go | 3 + .../memcached_server_selector_test.go | 3 + pkg/compact/clean.go | 3 + pkg/compact/clean_test.go | 3 + pkg/compact/compact.go | 3 + pkg/compact/compact_e2e_test.go | 3 + pkg/compact/compact_test.go | 3 + pkg/compact/downsample/aggr.go | 3 + pkg/compact/downsample/aggr_test.go | 3 + pkg/compact/downsample/downsample.go | 3 + pkg/compact/downsample/downsample_test.go | 3 + pkg/compact/downsample/pool.go | 3 + .../downsample/streamed_block_writer.go | 3 + pkg/compact/retention.go | 3 + pkg/compact/retention_test.go | 3 + pkg/component/component.go | 3 + pkg/discovery/cache/cache.go | 3 + pkg/discovery/cache/cache_test.go | 3 + pkg/discovery/dns/miekgdns/lookup.go | 3 + pkg/discovery/dns/miekgdns/resolver.go | 3 + pkg/discovery/dns/provider.go | 3 + pkg/discovery/dns/provider_test.go | 3 + pkg/discovery/dns/resolver.go | 3 + pkg/discovery/dns/resolver_test.go | 3 + pkg/extflag/pathorcontent.go | 3 + pkg/extgrpc/client.go | 3 + pkg/exthttp/transport.go | 3 + pkg/extprom/extprom.go | 3 + pkg/extprom/http/instrument_server.go | 3 + pkg/extprom/tx_gauge.go | 3 + pkg/extprom/tx_gauge_test.go | 3 + pkg/gate/gate.go | 3 + pkg/http/http.go | 3 + pkg/model/timeduration.go | 3 + pkg/model/timeduration_test.go | 3 + pkg/objstore/azure/azure.go | 3 + pkg/objstore/azure/azure_test.go | 3 + pkg/objstore/azure/helpers.go | 3 + pkg/objstore/azure/helpers_test.go | 3 + pkg/objstore/client/factory.go | 3 + pkg/objstore/cos/cos.go | 3 + pkg/objstore/filesystem/filesystem.go | 3 + pkg/objstore/gcs/gcs.go | 3 + pkg/objstore/inmem/inmem.go | 3 + pkg/objstore/objstore.go | 3 + .../objtesting/acceptance_e2e_test.go | 3 + pkg/objstore/objtesting/foreach.go | 3 + pkg/objstore/oss/oss.go | 3 + pkg/objstore/s3/s3.go | 3 + pkg/objstore/s3/s3_test.go | 3 + pkg/objstore/swift/swift.go | 3 + pkg/objstore/swift/swift_test.go | 3 + pkg/objstore/testing.go | 3 + pkg/pool/pool.go | 3 + pkg/pool/pool_test.go | 3 + pkg/prober/combiner.go | 3 + pkg/prober/grpc.go | 3 + pkg/prober/http.go | 3 + pkg/prober/http_test.go | 3 + pkg/prober/intrumentation.go | 3 + pkg/prober/prober.go | 3 + pkg/promclient/promclient.go | 3 + pkg/promclient/promclient_e2e_test.go | 3 + pkg/query/api/v1.go | 3 + pkg/query/api/v1_test.go | 3 + pkg/query/config.go | 3 + .../test-storeset-pre-v0.8.0/storeset.go | 3 + .../test-storeset-pre-v0.8.0/storeset_test.go | 3 + pkg/query/iter.go | 3 + pkg/query/querier.go | 3 + pkg/query/querier_test.go | 3 + pkg/query/storeset.go | 3 + pkg/query/storeset_test.go | 3 + pkg/receive/config.go | 3 + pkg/receive/handler.go | 3 + pkg/receive/handler_test.go | 3 + pkg/receive/hashring.go | 3 + pkg/receive/hashring_test.go | 3 + pkg/receive/tsdb.go | 3 + pkg/receive/tsdb_test.go | 3 + pkg/receive/writer.go | 3 + pkg/reloader/example_test.go | 3 + pkg/reloader/reloader.go | 3 + pkg/reloader/reloader_test.go | 3 + pkg/rule/api/v1.go | 3 + pkg/rule/api/v1_test.go | 3 + pkg/rule/rule.go | 3 + pkg/rule/rule_test.go | 3 + pkg/runutil/example_test.go | 3 + pkg/runutil/runutil.go | 3 + pkg/runutil/runutil_test.go | 3 + pkg/server/grpc/grpc.go | 3 + pkg/server/grpc/option.go | 3 + pkg/server/http/http.go | 3 + pkg/server/http/option.go | 3 + pkg/shipper/shipper.go | 3 + pkg/shipper/shipper_e2e_test.go | 3 + pkg/shipper/shipper_test.go | 3 + pkg/store/bucket.go | 3 + pkg/store/bucket_e2e_test.go | 3 + pkg/store/bucket_test.go | 3 + pkg/store/cache/cache.go | 3 + pkg/store/cache/cache_test.go | 3 + pkg/store/cache/factory.go | 3 + pkg/store/cache/inmemory.go | 3 + pkg/store/cache/inmemory_test.go | 3 + pkg/store/cache/memcached.go | 3 + pkg/store/cache/memcached_test.go | 3 + pkg/store/cache/units.go | 3 + pkg/store/cache/units_test.go | 3 + pkg/store/limiter.go | 3 + pkg/store/matchers.go | 3 + pkg/store/matchers_test.go | 3 + pkg/store/prometheus.go | 3 + pkg/store/prometheus_test.go | 3 + pkg/store/proxy.go | 3 + pkg/store/proxy_test.go | 3 + pkg/store/storepb/custom.go | 3 + pkg/store/storepb/custom_test.go | 3 + pkg/store/storepb/rpc.proto | 3 + pkg/store/storepb/types.proto | 3 + pkg/store/tsdb.go | 3 + pkg/store/tsdb_test.go | 3 + pkg/strutil/merge.go | 3 + pkg/testutil/copy.go | 3 + pkg/testutil/port.go | 3 + pkg/testutil/prometheus.go | 3 + pkg/testutil/sysprocattr.go | 3 + pkg/testutil/sysprocattr_linux.go | 3 + pkg/tls/options.go | 3 + pkg/tracing/client/factory.go | 3 + pkg/tracing/elasticapm/elastic_apm.go | 3 + pkg/tracing/grpc.go | 3 + pkg/tracing/http.go | 3 + pkg/tracing/jaeger/config_yaml.go | 3 + pkg/tracing/jaeger/jaeger.go | 3 + pkg/tracing/jaeger/logger.go | 3 + pkg/tracing/lightstep/lightstep.go | 3 + pkg/tracing/stackdriver/stackdriver.go | 3 + pkg/tracing/stackdriver/tracer.go | 3 + pkg/tracing/stackdriver/tracer_test.go | 3 + pkg/tracing/tracing.go | 3 + pkg/ui/bucket.go | 3 + pkg/ui/query.go | 3 + pkg/ui/rule.go | 3 + pkg/ui/ui.go | 3 + pkg/ui/ui_test.go | 3 + pkg/verifier/duplicated_compaction.go | 3 + pkg/verifier/duplicated_compaction_test.go | 3 + pkg/verifier/index_issue.go | 3 + pkg/verifier/overlapped_blocks.go | 3 + pkg/verifier/safe_delete.go | 3 + pkg/verifier/verify.go | 3 + scripts/build-check-comments.sh | 5 +- scripts/cfggen/main.go | 3 + scripts/copyright/copyright.go | 71 +++++++++++++++++++ test/e2e/query_test.go | 3 + test/e2e/receive_test.go | 3 + test/e2e/rule_test.go | 3 + test/e2e/spinup_test.go | 3 + test/e2e/store_gateway_test.go | 3 + 195 files changed, 654 insertions(+), 1 deletion(-) create mode 100644 scripts/copyright/copyright.go diff --git a/Makefile b/Makefile index cbdf5ef39a..07acaaf1f5 100644 --- a/Makefile +++ b/Makefile @@ -299,6 +299,9 @@ lint: check-git $(GOLANGCILINT) $(MISSPELL) @echo ">> detecting white noise" @find . -type f \( -name "*.md" -o -name "*.go" \) | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh $(call require_clean_work_tree,"detected white noise") + @echo ">> ensuring Copyright headers" + @go run ./scripts/copyright + $(call require_clean_work_tree,"detected files without copyright") .PHONY: web-serve web-serve: web-pre-process $(HUGO) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index b7bb9ea584..a6a5cb771b 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/check.go b/cmd/thanos/check.go index bb85361872..d42c12052c 100644 --- a/cmd/thanos/check.go +++ b/cmd/thanos/check.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/check_test.go b/cmd/thanos/check_test.go index 39c3a360ca..299b0276e5 100644 --- a/cmd/thanos/check_test.go +++ b/cmd/thanos/check_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 084630971a..4bdd9cc351 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index bf5cf7e58f..b7b65369fa 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/flags.go b/cmd/thanos/flags.go index b6d7776086..fac85c125a 100644 --- a/cmd/thanos/flags.go +++ b/cmd/thanos/flags.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/main.go b/cmd/thanos/main.go index e4b7a06e72..c086782a7d 100644 --- a/cmd/thanos/main.go +++ b/cmd/thanos/main.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index f6aedb48eb..839c125e07 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 159a3ef16e..f64584b27d 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 71b2194c91..6f5e238618 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index db7edd7144..7357f7ae48 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/rule_test.go b/cmd/thanos/rule_test.go index 2abd38cf6a..96a9bc66b1 100644 --- a/cmd/thanos/rule_test.go +++ b/cmd/thanos/rule_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 0b7babf2d1..14d559c49f 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index dc3a0cf225..b6b7ed320c 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/doc.go b/doc.go index c8397f280b..d1cd0c6779 100644 --- a/doc.go +++ b/doc.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package thanos is a set of components that // can provide highly available Prometheus // setup with long term storage capabilities. diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go index 82b9602d7a..7cb11a75bd 100644 --- a/pkg/alert/alert.go +++ b/pkg/alert/alert.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package alert contains logic to send alert notifications to Alertmanager clusters. package alert diff --git a/pkg/alert/alert_test.go b/pkg/alert/alert_test.go index 3b99266173..7139dfc652 100644 --- a/pkg/alert/alert_test.go +++ b/pkg/alert/alert_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package alert import ( diff --git a/pkg/alert/config.go b/pkg/alert/config.go index 6e402091d4..b701b39292 100644 --- a/pkg/alert/config.go +++ b/pkg/alert/config.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package alert import ( diff --git a/pkg/alert/config_test.go b/pkg/alert/config_test.go index ea7c3de773..71aaee399c 100644 --- a/pkg/alert/config_test.go +++ b/pkg/alert/config_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package alert import ( diff --git a/pkg/block/block.go b/pkg/block/block.go index 147a82f068..740b3d6306 100644 --- a/pkg/block/block.go +++ b/pkg/block/block.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package block contains common functionality for interacting with TSDB blocks // in the context of Thanos. package block diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index 35d22871f5..38b649afe2 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package block import ( diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index 3fc658e3ba..6250d01111 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package block import ( diff --git a/pkg/block/fetcher_test.go b/pkg/block/fetcher_test.go index 8be50a928f..faf7eafb5a 100644 --- a/pkg/block/fetcher_test.go +++ b/pkg/block/fetcher_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package block import ( diff --git a/pkg/block/index.go b/pkg/block/index.go index 0df14ded4d..2551aacd9c 100644 --- a/pkg/block/index.go +++ b/pkg/block/index.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package block import ( diff --git a/pkg/block/index_test.go b/pkg/block/index_test.go index 257005c7e2..57860461b7 100644 --- a/pkg/block/index_test.go +++ b/pkg/block/index_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package block import ( diff --git a/pkg/block/indexheader/binary_reader.go b/pkg/block/indexheader/binary_reader.go index 095432a80a..3466046371 100644 --- a/pkg/block/indexheader/binary_reader.go +++ b/pkg/block/indexheader/binary_reader.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package indexheader import ( diff --git a/pkg/block/indexheader/header.go b/pkg/block/indexheader/header.go index 04b3f14049..dbbe335deb 100644 --- a/pkg/block/indexheader/header.go +++ b/pkg/block/indexheader/header.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package indexheader import ( diff --git a/pkg/block/indexheader/header_test.go b/pkg/block/indexheader/header_test.go index 0fb8af221e..932071482c 100644 --- a/pkg/block/indexheader/header_test.go +++ b/pkg/block/indexheader/header_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package indexheader import ( diff --git a/pkg/block/indexheader/json_reader.go b/pkg/block/indexheader/json_reader.go index 67b502e791..d8ef13193f 100644 --- a/pkg/block/indexheader/json_reader.go +++ b/pkg/block/indexheader/json_reader.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package indexheader import ( diff --git a/pkg/block/metadata/meta.go b/pkg/block/metadata/meta.go index fd664c1aa3..b57b57722f 100644 --- a/pkg/block/metadata/meta.go +++ b/pkg/block/metadata/meta.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package metadata // metadata package implements writing and reading wrapped meta.json where Thanos puts its metadata. diff --git a/pkg/cacheutil/jump_hash.go b/pkg/cacheutil/jump_hash.go index 4b684b7384..cd962773f9 100644 --- a/pkg/cacheutil/jump_hash.go +++ b/pkg/cacheutil/jump_hash.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cacheutil // jumpHash consistently chooses a hash bucket number in the range diff --git a/pkg/cacheutil/memcached_client.go b/pkg/cacheutil/memcached_client.go index 91d4552dd1..2785f59d00 100644 --- a/pkg/cacheutil/memcached_client.go +++ b/pkg/cacheutil/memcached_client.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cacheutil import ( diff --git a/pkg/cacheutil/memcached_client_test.go b/pkg/cacheutil/memcached_client_test.go index a1f77fe282..cfba6fb4ad 100644 --- a/pkg/cacheutil/memcached_client_test.go +++ b/pkg/cacheutil/memcached_client_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cacheutil import ( diff --git a/pkg/cacheutil/memcached_server_selector.go b/pkg/cacheutil/memcached_server_selector.go index 37b693367c..00a122e075 100644 --- a/pkg/cacheutil/memcached_server_selector.go +++ b/pkg/cacheutil/memcached_server_selector.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cacheutil import ( diff --git a/pkg/cacheutil/memcached_server_selector_test.go b/pkg/cacheutil/memcached_server_selector_test.go index 466db1c401..a8da4346ca 100644 --- a/pkg/cacheutil/memcached_server_selector_test.go +++ b/pkg/cacheutil/memcached_server_selector_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cacheutil import ( diff --git a/pkg/compact/clean.go b/pkg/compact/clean.go index 7fa085eebc..af681a940e 100644 --- a/pkg/compact/clean.go +++ b/pkg/compact/clean.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package compact import ( diff --git a/pkg/compact/clean_test.go b/pkg/compact/clean_test.go index 5332da7637..85654f8c8a 100644 --- a/pkg/compact/clean_test.go +++ b/pkg/compact/clean_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package compact import ( diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 49d641d0b3..8c02f11c00 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package compact import ( diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 926175dc68..80dba2a795 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package compact import ( diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index f7364f1147..a3c51f61c0 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package compact import ( diff --git a/pkg/compact/downsample/aggr.go b/pkg/compact/downsample/aggr.go index 9a96e94571..db5f914e12 100644 --- a/pkg/compact/downsample/aggr.go +++ b/pkg/compact/downsample/aggr.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package downsample import ( diff --git a/pkg/compact/downsample/aggr_test.go b/pkg/compact/downsample/aggr_test.go index 55fa4ac0f8..62a1b7fe1a 100644 --- a/pkg/compact/downsample/aggr_test.go +++ b/pkg/compact/downsample/aggr_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package downsample import ( diff --git a/pkg/compact/downsample/downsample.go b/pkg/compact/downsample/downsample.go index 5bcfccc81b..5e14c26bb3 100644 --- a/pkg/compact/downsample/downsample.go +++ b/pkg/compact/downsample/downsample.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package downsample import ( diff --git a/pkg/compact/downsample/downsample_test.go b/pkg/compact/downsample/downsample_test.go index 9b13acf652..34666de66c 100644 --- a/pkg/compact/downsample/downsample_test.go +++ b/pkg/compact/downsample/downsample_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package downsample import ( diff --git a/pkg/compact/downsample/pool.go b/pkg/compact/downsample/pool.go index f591b16548..68dec1227e 100644 --- a/pkg/compact/downsample/pool.go +++ b/pkg/compact/downsample/pool.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package downsample import ( diff --git a/pkg/compact/downsample/streamed_block_writer.go b/pkg/compact/downsample/streamed_block_writer.go index bde6397869..1cfad55749 100644 --- a/pkg/compact/downsample/streamed_block_writer.go +++ b/pkg/compact/downsample/streamed_block_writer.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package downsample import ( diff --git a/pkg/compact/retention.go b/pkg/compact/retention.go index aba46963ec..47f61f0cfb 100644 --- a/pkg/compact/retention.go +++ b/pkg/compact/retention.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package compact import ( diff --git a/pkg/compact/retention_test.go b/pkg/compact/retention_test.go index b34847faf3..371d3c4152 100644 --- a/pkg/compact/retention_test.go +++ b/pkg/compact/retention_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package compact_test import ( diff --git a/pkg/component/component.go b/pkg/component/component.go index d00966fc2e..675caa6034 100644 --- a/pkg/component/component.go +++ b/pkg/component/component.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package component import ( diff --git a/pkg/discovery/cache/cache.go b/pkg/discovery/cache/cache.go index c619eaa793..172547771f 100644 --- a/pkg/discovery/cache/cache.go +++ b/pkg/discovery/cache/cache.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cache import ( diff --git a/pkg/discovery/cache/cache_test.go b/pkg/discovery/cache/cache_test.go index 695eef84e3..f6b6959cb7 100644 --- a/pkg/discovery/cache/cache_test.go +++ b/pkg/discovery/cache/cache_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cache import ( diff --git a/pkg/discovery/dns/miekgdns/lookup.go b/pkg/discovery/dns/miekgdns/lookup.go index 831eae64d7..4e3fb492e2 100644 --- a/pkg/discovery/dns/miekgdns/lookup.go +++ b/pkg/discovery/dns/miekgdns/lookup.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package miekgdns import ( diff --git a/pkg/discovery/dns/miekgdns/resolver.go b/pkg/discovery/dns/miekgdns/resolver.go index 06b0b25a1a..e62660f12c 100644 --- a/pkg/discovery/dns/miekgdns/resolver.go +++ b/pkg/discovery/dns/miekgdns/resolver.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package miekgdns import ( diff --git a/pkg/discovery/dns/provider.go b/pkg/discovery/dns/provider.go index 332135ba17..a3e0730730 100644 --- a/pkg/discovery/dns/provider.go +++ b/pkg/discovery/dns/provider.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package dns import ( diff --git a/pkg/discovery/dns/provider_test.go b/pkg/discovery/dns/provider_test.go index 0483a023e5..3a05c06a67 100644 --- a/pkg/discovery/dns/provider_test.go +++ b/pkg/discovery/dns/provider_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package dns import ( diff --git a/pkg/discovery/dns/resolver.go b/pkg/discovery/dns/resolver.go index 709e264e57..ef73054768 100644 --- a/pkg/discovery/dns/resolver.go +++ b/pkg/discovery/dns/resolver.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package dns import ( diff --git a/pkg/discovery/dns/resolver_test.go b/pkg/discovery/dns/resolver_test.go index 7209e031b9..f7d271d0c8 100644 --- a/pkg/discovery/dns/resolver_test.go +++ b/pkg/discovery/dns/resolver_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package dns import ( diff --git a/pkg/extflag/pathorcontent.go b/pkg/extflag/pathorcontent.go index 8b22bee714..bc2e82b8ac 100644 --- a/pkg/extflag/pathorcontent.go +++ b/pkg/extflag/pathorcontent.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package extflag import ( diff --git a/pkg/extgrpc/client.go b/pkg/extgrpc/client.go index 2d97f9894b..eb6ac47595 100644 --- a/pkg/extgrpc/client.go +++ b/pkg/extgrpc/client.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package extgrpc import ( diff --git a/pkg/exthttp/transport.go b/pkg/exthttp/transport.go index 92725c9b33..60e82e2cc8 100644 --- a/pkg/exthttp/transport.go +++ b/pkg/exthttp/transport.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package exthttp import ( diff --git a/pkg/extprom/extprom.go b/pkg/extprom/extprom.go index f2d7d09ab1..5da95f3b17 100644 --- a/pkg/extprom/extprom.go +++ b/pkg/extprom/extprom.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package extprom import "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/extprom/http/instrument_server.go b/pkg/extprom/http/instrument_server.go index aaef94693b..907d3da6e5 100644 --- a/pkg/extprom/http/instrument_server.go +++ b/pkg/extprom/http/instrument_server.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package http import ( diff --git a/pkg/extprom/tx_gauge.go b/pkg/extprom/tx_gauge.go index d0ee7342e6..d85bf4f921 100644 --- a/pkg/extprom/tx_gauge.go +++ b/pkg/extprom/tx_gauge.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package extprom import ( diff --git a/pkg/extprom/tx_gauge_test.go b/pkg/extprom/tx_gauge_test.go index a2c3a5988c..4be8b02de4 100644 --- a/pkg/extprom/tx_gauge_test.go +++ b/pkg/extprom/tx_gauge_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package extprom import ( diff --git a/pkg/gate/gate.go b/pkg/gate/gate.go index 78aeefe671..6b7dbfc448 100644 --- a/pkg/gate/gate.go +++ b/pkg/gate/gate.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package gate import ( diff --git a/pkg/http/http.go b/pkg/http/http.go index 8686152139..27fb25234b 100644 --- a/pkg/http/http.go +++ b/pkg/http/http.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package http is a wrapper around github.com/prometheus/common/config. package http diff --git a/pkg/model/timeduration.go b/pkg/model/timeduration.go index a40a6a78f4..1d525136f0 100644 --- a/pkg/model/timeduration.go +++ b/pkg/model/timeduration.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package model import ( diff --git a/pkg/model/timeduration_test.go b/pkg/model/timeduration_test.go index 30f2f71495..c1b0a72a73 100644 --- a/pkg/model/timeduration_test.go +++ b/pkg/model/timeduration_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package model_test import ( diff --git a/pkg/objstore/azure/azure.go b/pkg/objstore/azure/azure.go index b5ef93ab3d..0c132daf61 100644 --- a/pkg/objstore/azure/azure.go +++ b/pkg/objstore/azure/azure.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package azure import ( diff --git a/pkg/objstore/azure/azure_test.go b/pkg/objstore/azure/azure_test.go index 280be7fb98..5c30e73757 100644 --- a/pkg/objstore/azure/azure_test.go +++ b/pkg/objstore/azure/azure_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package azure import ( diff --git a/pkg/objstore/azure/helpers.go b/pkg/objstore/azure/helpers.go index 86a76f0911..0189a49acf 100644 --- a/pkg/objstore/azure/helpers.go +++ b/pkg/objstore/azure/helpers.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package azure import ( diff --git a/pkg/objstore/azure/helpers_test.go b/pkg/objstore/azure/helpers_test.go index b44ba5d32d..3b1b22119d 100644 --- a/pkg/objstore/azure/helpers_test.go +++ b/pkg/objstore/azure/helpers_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package azure import ( diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 145b2157a1..6caf7f8ee4 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package client import ( diff --git a/pkg/objstore/cos/cos.go b/pkg/objstore/cos/cos.go index df4fc04ee0..65768d46b1 100644 --- a/pkg/objstore/cos/cos.go +++ b/pkg/objstore/cos/cos.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cos import ( diff --git a/pkg/objstore/filesystem/filesystem.go b/pkg/objstore/filesystem/filesystem.go index cac855588a..0a13994d93 100644 --- a/pkg/objstore/filesystem/filesystem.go +++ b/pkg/objstore/filesystem/filesystem.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package filesystem import ( diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index 533659dc28..ae2cf3202b 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package gcs implements common object storage abstractions against Google Cloud Storage. package gcs diff --git a/pkg/objstore/inmem/inmem.go b/pkg/objstore/inmem/inmem.go index 8dd391d570..7f430c3c97 100644 --- a/pkg/objstore/inmem/inmem.go +++ b/pkg/objstore/inmem/inmem.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package inmem import ( diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index ba6e775f82..e187be3c62 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package objstore import ( diff --git a/pkg/objstore/objtesting/acceptance_e2e_test.go b/pkg/objstore/objtesting/acceptance_e2e_test.go index 1658aca397..86045424ee 100644 --- a/pkg/objstore/objtesting/acceptance_e2e_test.go +++ b/pkg/objstore/objtesting/acceptance_e2e_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package objtesting import ( diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go index 07d4b352d2..97b0490977 100644 --- a/pkg/objstore/objtesting/foreach.go +++ b/pkg/objstore/objtesting/foreach.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package objtesting import ( diff --git a/pkg/objstore/oss/oss.go b/pkg/objstore/oss/oss.go index ff5ab1831f..effc1e217a 100644 --- a/pkg/objstore/oss/oss.go +++ b/pkg/objstore/oss/oss.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package oss import ( diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 2ca9d187c4..d924eeb01f 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package s3 implements common object storage abstractions against s3-compatible APIs. package s3 diff --git a/pkg/objstore/s3/s3_test.go b/pkg/objstore/s3/s3_test.go index 1b9b75d457..3ef3ea19e2 100644 --- a/pkg/objstore/s3/s3_test.go +++ b/pkg/objstore/s3/s3_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package s3 import ( diff --git a/pkg/objstore/swift/swift.go b/pkg/objstore/swift/swift.go index 37bff9e6aa..676c3f7d59 100644 --- a/pkg/objstore/swift/swift.go +++ b/pkg/objstore/swift/swift.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package swift implements common object storage abstractions against OpenStack swift APIs. package swift diff --git a/pkg/objstore/swift/swift_test.go b/pkg/objstore/swift/swift_test.go index a45961683c..0a789cc8df 100644 --- a/pkg/objstore/swift/swift_test.go +++ b/pkg/objstore/swift/swift_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package swift import ( diff --git a/pkg/objstore/testing.go b/pkg/objstore/testing.go index 18010aa5f3..9e9a864b91 100644 --- a/pkg/objstore/testing.go +++ b/pkg/objstore/testing.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package objstore import ( diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go index 9c4a2f2212..939628acbf 100644 --- a/pkg/pool/pool.go +++ b/pkg/pool/pool.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package pool import ( diff --git a/pkg/pool/pool_test.go b/pkg/pool/pool_test.go index 616010c803..955e85b08b 100644 --- a/pkg/pool/pool_test.go +++ b/pkg/pool/pool_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package pool import ( diff --git a/pkg/prober/combiner.go b/pkg/prober/combiner.go index b85d6a195e..348e71cf4c 100644 --- a/pkg/prober/combiner.go +++ b/pkg/prober/combiner.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package prober import "sync" diff --git a/pkg/prober/grpc.go b/pkg/prober/grpc.go index 117ced0427..6151e04033 100644 --- a/pkg/prober/grpc.go +++ b/pkg/prober/grpc.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package prober import ( diff --git a/pkg/prober/http.go b/pkg/prober/http.go index 96777a09b2..fe273741f1 100644 --- a/pkg/prober/http.go +++ b/pkg/prober/http.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package prober import ( diff --git a/pkg/prober/http_test.go b/pkg/prober/http_test.go index 955cbd3921..e31368a0f3 100644 --- a/pkg/prober/http_test.go +++ b/pkg/prober/http_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package prober import ( diff --git a/pkg/prober/intrumentation.go b/pkg/prober/intrumentation.go index 8a9751de93..076996d88f 100644 --- a/pkg/prober/intrumentation.go +++ b/pkg/prober/intrumentation.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package prober import ( diff --git a/pkg/prober/prober.go b/pkg/prober/prober.go index 87bff192e3..e4344bf45e 100644 --- a/pkg/prober/prober.go +++ b/pkg/prober/prober.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package prober // Prober represents health and readiness status of given component. diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index b80e6819d9..d5f27e32b6 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package promclient offers helper client function for various API endpoints. package promclient diff --git a/pkg/promclient/promclient_e2e_test.go b/pkg/promclient/promclient_e2e_test.go index adb6bbd1b7..61f5a5e85a 100644 --- a/pkg/promclient/promclient_e2e_test.go +++ b/pkg/promclient/promclient_e2e_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package promclient import ( diff --git a/pkg/query/api/v1.go b/pkg/query/api/v1.go index 0fb1b24b44..74d0effc8c 100644 --- a/pkg/query/api/v1.go +++ b/pkg/query/api/v1.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Copyright 2016 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/query/api/v1_test.go b/pkg/query/api/v1_test.go index c4be888a24..80b602bec7 100644 --- a/pkg/query/api/v1_test.go +++ b/pkg/query/api/v1_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Copyright 2016 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/query/config.go b/pkg/query/config.go index c4c918b74e..3787695410 100644 --- a/pkg/query/config.go +++ b/pkg/query/config.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package query import ( diff --git a/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go b/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go index 254491b690..326c50b80b 100644 --- a/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go +++ b/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + /* This package is for compatibility testing purposes. It is a code from v0.7.0 Querier. */ diff --git a/pkg/query/internal/test-storeset-pre-v0.8.0/storeset_test.go b/pkg/query/internal/test-storeset-pre-v0.8.0/storeset_test.go index 37b94836b1..14b3635a80 100644 --- a/pkg/query/internal/test-storeset-pre-v0.8.0/storeset_test.go +++ b/pkg/query/internal/test-storeset-pre-v0.8.0/storeset_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package testoldstoreset import ( diff --git a/pkg/query/iter.go b/pkg/query/iter.go index 3d421c3f0c..a994b8216c 100644 --- a/pkg/query/iter.go +++ b/pkg/query/iter.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package query import ( diff --git a/pkg/query/querier.go b/pkg/query/querier.go index a8294b00fa..016a7ab442 100644 --- a/pkg/query/querier.go +++ b/pkg/query/querier.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package query import ( diff --git a/pkg/query/querier_test.go b/pkg/query/querier_test.go index 57ea33bea9..f5cf7a9558 100644 --- a/pkg/query/querier_test.go +++ b/pkg/query/querier_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package query import ( diff --git a/pkg/query/storeset.go b/pkg/query/storeset.go index 5f478c5d42..978a3485ed 100644 --- a/pkg/query/storeset.go +++ b/pkg/query/storeset.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package query import ( diff --git a/pkg/query/storeset_test.go b/pkg/query/storeset_test.go index 6deba8e423..b3dd853dd5 100644 --- a/pkg/query/storeset_test.go +++ b/pkg/query/storeset_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package query import ( diff --git a/pkg/receive/config.go b/pkg/receive/config.go index 24be94b856..8dc945bd79 100644 --- a/pkg/receive/config.go +++ b/pkg/receive/config.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package receive import ( diff --git a/pkg/receive/handler.go b/pkg/receive/handler.go index 5ee91d1017..4be160972a 100644 --- a/pkg/receive/handler.go +++ b/pkg/receive/handler.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package receive import ( diff --git a/pkg/receive/handler_test.go b/pkg/receive/handler_test.go index 6f58b84cec..5d48e32f21 100644 --- a/pkg/receive/handler_test.go +++ b/pkg/receive/handler_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package receive import ( diff --git a/pkg/receive/hashring.go b/pkg/receive/hashring.go index b755756a2b..947ff0d1fd 100644 --- a/pkg/receive/hashring.go +++ b/pkg/receive/hashring.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package receive import ( diff --git a/pkg/receive/hashring_test.go b/pkg/receive/hashring_test.go index c16334a24e..b2a50fdc4b 100644 --- a/pkg/receive/hashring_test.go +++ b/pkg/receive/hashring_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package receive import ( diff --git a/pkg/receive/tsdb.go b/pkg/receive/tsdb.go index 8a081f0601..55fac10169 100644 --- a/pkg/receive/tsdb.go +++ b/pkg/receive/tsdb.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package receive import ( diff --git a/pkg/receive/tsdb_test.go b/pkg/receive/tsdb_test.go index 75a80f1b59..5a527bef0f 100644 --- a/pkg/receive/tsdb_test.go +++ b/pkg/receive/tsdb_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package receive import ( diff --git a/pkg/receive/writer.go b/pkg/receive/writer.go index 3a8a20f827..aeccbfbfbe 100644 --- a/pkg/receive/writer.go +++ b/pkg/receive/writer.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package receive import ( diff --git a/pkg/reloader/example_test.go b/pkg/reloader/example_test.go index 78d4b99040..5f67a5caf3 100644 --- a/pkg/reloader/example_test.go +++ b/pkg/reloader/example_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package reloader_test import ( diff --git a/pkg/reloader/reloader.go b/pkg/reloader/reloader.go index e51edd7d37..508c14280f 100644 --- a/pkg/reloader/reloader.go +++ b/pkg/reloader/reloader.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package reloader contains helpers to trigger reloads of Prometheus instances // on configuration changes and to substitute environment variables in config files. // diff --git a/pkg/reloader/reloader_test.go b/pkg/reloader/reloader_test.go index fb6ede8cee..3784635349 100644 --- a/pkg/reloader/reloader_test.go +++ b/pkg/reloader/reloader_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package reloader import ( diff --git a/pkg/rule/api/v1.go b/pkg/rule/api/v1.go index 7927a97d67..682a79d5e7 100644 --- a/pkg/rule/api/v1.go +++ b/pkg/rule/api/v1.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package v1 import ( diff --git a/pkg/rule/api/v1_test.go b/pkg/rule/api/v1_test.go index ce71f5d16b..4d14f68c73 100644 --- a/pkg/rule/api/v1_test.go +++ b/pkg/rule/api/v1_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package v1 import ( diff --git a/pkg/rule/rule.go b/pkg/rule/rule.go index a473b517f4..8491478c13 100644 --- a/pkg/rule/rule.go +++ b/pkg/rule/rule.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package thanosrule import ( diff --git a/pkg/rule/rule_test.go b/pkg/rule/rule_test.go index 0e1cc9ad0b..d238db10d4 100644 --- a/pkg/rule/rule_test.go +++ b/pkg/rule/rule_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package thanosrule import ( diff --git a/pkg/runutil/example_test.go b/pkg/runutil/example_test.go index cc6b859498..5f15aefaa0 100644 --- a/pkg/runutil/example_test.go +++ b/pkg/runutil/example_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package runutil_test import ( diff --git a/pkg/runutil/runutil.go b/pkg/runutil/runutil.go index e5f7fa2ef6..45d06f4e08 100644 --- a/pkg/runutil/runutil.go +++ b/pkg/runutil/runutil.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package runutil provides helpers to advanced function scheduling control like repeat or retry. // // It's very often the case when you need to excutes some code every fixed intervals or have it retried automatically. diff --git a/pkg/runutil/runutil_test.go b/pkg/runutil/runutil_test.go index 06c13151f9..bdbebdeb25 100644 --- a/pkg/runutil/runutil_test.go +++ b/pkg/runutil/runutil_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package runutil import ( diff --git a/pkg/server/grpc/grpc.go b/pkg/server/grpc/grpc.go index b1b7779541..34e25a90f2 100644 --- a/pkg/server/grpc/grpc.go +++ b/pkg/server/grpc/grpc.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package grpc import ( diff --git a/pkg/server/grpc/option.go b/pkg/server/grpc/option.go index 6c804e4f79..6df898ae16 100644 --- a/pkg/server/grpc/option.go +++ b/pkg/server/grpc/option.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package grpc import ( diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index 52e839d9f5..162da79a77 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package http import ( diff --git a/pkg/server/http/option.go b/pkg/server/http/option.go index ab7ff59e0c..398cabd93d 100644 --- a/pkg/server/http/option.go +++ b/pkg/server/http/option.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package http import ( diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index 00ced432db..39e7d61d86 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Package shipper detects directories on the local file system and uploads // them to a block storage. package shipper diff --git a/pkg/shipper/shipper_e2e_test.go b/pkg/shipper/shipper_e2e_test.go index d84e1b6d16..2b17e460e7 100644 --- a/pkg/shipper/shipper_e2e_test.go +++ b/pkg/shipper/shipper_e2e_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package shipper import ( diff --git a/pkg/shipper/shipper_test.go b/pkg/shipper/shipper_test.go index 85a78b16dc..fd9b75706f 100644 --- a/pkg/shipper/shipper_test.go +++ b/pkg/shipper/shipper_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package shipper import ( diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 48636be7c7..bdeaaf2f9e 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index 531cf73552..77bd8dd21a 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index dbef61e29a..216adedc09 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go index d7e175da82..2973de7b22 100644 --- a/pkg/store/cache/cache.go +++ b/pkg/store/cache/cache.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storecache import ( diff --git a/pkg/store/cache/cache_test.go b/pkg/store/cache/cache_test.go index 60b0a3db7f..670fd4d133 100644 --- a/pkg/store/cache/cache_test.go +++ b/pkg/store/cache/cache_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storecache import ( diff --git a/pkg/store/cache/factory.go b/pkg/store/cache/factory.go index 7fed27dac2..e67251d3c7 100644 --- a/pkg/store/cache/factory.go +++ b/pkg/store/cache/factory.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storecache import ( diff --git a/pkg/store/cache/inmemory.go b/pkg/store/cache/inmemory.go index 3b8881eb65..02d5ad12ba 100644 --- a/pkg/store/cache/inmemory.go +++ b/pkg/store/cache/inmemory.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storecache import ( diff --git a/pkg/store/cache/inmemory_test.go b/pkg/store/cache/inmemory_test.go index e14d3794af..460c5c8c73 100644 --- a/pkg/store/cache/inmemory_test.go +++ b/pkg/store/cache/inmemory_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // Tests out the index cache implementation. package storecache diff --git a/pkg/store/cache/memcached.go b/pkg/store/cache/memcached.go index 51c028dd54..db7ec61c63 100644 --- a/pkg/store/cache/memcached.go +++ b/pkg/store/cache/memcached.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storecache import ( diff --git a/pkg/store/cache/memcached_test.go b/pkg/store/cache/memcached_test.go index 83d858ff30..c44b641758 100644 --- a/pkg/store/cache/memcached_test.go +++ b/pkg/store/cache/memcached_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storecache import ( diff --git a/pkg/store/cache/units.go b/pkg/store/cache/units.go index d9b6dce367..156ea54ddb 100644 --- a/pkg/store/cache/units.go +++ b/pkg/store/cache/units.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storecache import ( diff --git a/pkg/store/cache/units_test.go b/pkg/store/cache/units_test.go index 95438eaa50..bde202a2f1 100644 --- a/pkg/store/cache/units_test.go +++ b/pkg/store/cache/units_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storecache import ( diff --git a/pkg/store/limiter.go b/pkg/store/limiter.go index 2c332a2c6b..fc7216549c 100644 --- a/pkg/store/limiter.go +++ b/pkg/store/limiter.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/matchers.go b/pkg/store/matchers.go index 66902f5116..18abe0fba1 100644 --- a/pkg/store/matchers.go +++ b/pkg/store/matchers.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/matchers_test.go b/pkg/store/matchers_test.go index 75caf1791b..23097cbc7a 100644 --- a/pkg/store/matchers_test.go +++ b/pkg/store/matchers_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 976b33f47f..7d8c762421 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index a3adf05218..d2bb97b06c 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index b38f2c952b..9d2c677548 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/proxy_test.go b/pkg/store/proxy_test.go index ce26e2ba41..0e5af9492f 100644 --- a/pkg/store/proxy_test.go +++ b/pkg/store/proxy_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/storepb/custom.go b/pkg/store/storepb/custom.go index c942dd1355..ea13089be4 100644 --- a/pkg/store/storepb/custom.go +++ b/pkg/store/storepb/custom.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storepb import ( diff --git a/pkg/store/storepb/custom_test.go b/pkg/store/storepb/custom_test.go index 91865b5904..fe05a44b77 100644 --- a/pkg/store/storepb/custom_test.go +++ b/pkg/store/storepb/custom_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package storepb import ( diff --git a/pkg/store/storepb/rpc.proto b/pkg/store/storepb/rpc.proto index ebd3bb20c7..c314c40f08 100644 --- a/pkg/store/storepb/rpc.proto +++ b/pkg/store/storepb/rpc.proto @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + syntax = "proto3"; package thanos; diff --git a/pkg/store/storepb/types.proto b/pkg/store/storepb/types.proto index 24c9f4b318..635aa50469 100644 --- a/pkg/store/storepb/types.proto +++ b/pkg/store/storepb/types.proto @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + syntax = "proto3"; package thanos; diff --git a/pkg/store/tsdb.go b/pkg/store/tsdb.go index 3ea87412a6..076af640fe 100644 --- a/pkg/store/tsdb.go +++ b/pkg/store/tsdb.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/store/tsdb_test.go b/pkg/store/tsdb_test.go index 6e0354df36..d8ef8eda0c 100644 --- a/pkg/store/tsdb_test.go +++ b/pkg/store/tsdb_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package store import ( diff --git a/pkg/strutil/merge.go b/pkg/strutil/merge.go index 7a7d4f9509..d6108771f4 100644 --- a/pkg/strutil/merge.go +++ b/pkg/strutil/merge.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package strutil import ( diff --git a/pkg/testutil/copy.go b/pkg/testutil/copy.go index c2c8bd480d..f90dc68bbf 100644 --- a/pkg/testutil/copy.go +++ b/pkg/testutil/copy.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package testutil import ( diff --git a/pkg/testutil/port.go b/pkg/testutil/port.go index 3802da2d6e..482ced0858 100644 --- a/pkg/testutil/port.go +++ b/pkg/testutil/port.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package testutil import "net" diff --git a/pkg/testutil/prometheus.go b/pkg/testutil/prometheus.go index 505320fcaa..10f34b8a56 100644 --- a/pkg/testutil/prometheus.go +++ b/pkg/testutil/prometheus.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package testutil import ( diff --git a/pkg/testutil/sysprocattr.go b/pkg/testutil/sysprocattr.go index 7173d8d5f1..98d8755ce4 100644 --- a/pkg/testutil/sysprocattr.go +++ b/pkg/testutil/sysprocattr.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // +build !linux package testutil diff --git a/pkg/testutil/sysprocattr_linux.go b/pkg/testutil/sysprocattr_linux.go index 9670603018..f7c59a0b50 100644 --- a/pkg/testutil/sysprocattr_linux.go +++ b/pkg/testutil/sysprocattr_linux.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package testutil import "syscall" diff --git a/pkg/tls/options.go b/pkg/tls/options.go index 58b6c56d36..b7e102cccf 100644 --- a/pkg/tls/options.go +++ b/pkg/tls/options.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package tls import ( diff --git a/pkg/tracing/client/factory.go b/pkg/tracing/client/factory.go index 54f40a7292..b2e4a4f313 100644 --- a/pkg/tracing/client/factory.go +++ b/pkg/tracing/client/factory.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package client import ( diff --git a/pkg/tracing/elasticapm/elastic_apm.go b/pkg/tracing/elasticapm/elastic_apm.go index 0452ca1e7b..8b75c72a51 100644 --- a/pkg/tracing/elasticapm/elastic_apm.go +++ b/pkg/tracing/elasticapm/elastic_apm.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package elasticapm import ( diff --git a/pkg/tracing/grpc.go b/pkg/tracing/grpc.go index 91c861ebb3..303c387a67 100644 --- a/pkg/tracing/grpc.go +++ b/pkg/tracing/grpc.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package tracing import ( diff --git a/pkg/tracing/http.go b/pkg/tracing/http.go index 1344670fff..f158f0e6f3 100644 --- a/pkg/tracing/http.go +++ b/pkg/tracing/http.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package tracing import ( diff --git a/pkg/tracing/jaeger/config_yaml.go b/pkg/tracing/jaeger/config_yaml.go index 4139cd3d24..27d62e501d 100644 --- a/pkg/tracing/jaeger/config_yaml.go +++ b/pkg/tracing/jaeger/config_yaml.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package jaeger import ( diff --git a/pkg/tracing/jaeger/jaeger.go b/pkg/tracing/jaeger/jaeger.go index 73c65baf32..d1fa0e24ed 100644 --- a/pkg/tracing/jaeger/jaeger.go +++ b/pkg/tracing/jaeger/jaeger.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package jaeger import ( diff --git a/pkg/tracing/jaeger/logger.go b/pkg/tracing/jaeger/logger.go index 50b000197c..e30b67af28 100644 --- a/pkg/tracing/jaeger/logger.go +++ b/pkg/tracing/jaeger/logger.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package jaeger import ( diff --git a/pkg/tracing/lightstep/lightstep.go b/pkg/tracing/lightstep/lightstep.go index c6c3584905..a0fbf42976 100644 --- a/pkg/tracing/lightstep/lightstep.go +++ b/pkg/tracing/lightstep/lightstep.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package lightstep import ( diff --git a/pkg/tracing/stackdriver/stackdriver.go b/pkg/tracing/stackdriver/stackdriver.go index be2d9eb039..a35e5619f8 100644 --- a/pkg/tracing/stackdriver/stackdriver.go +++ b/pkg/tracing/stackdriver/stackdriver.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package stackdriver import ( diff --git a/pkg/tracing/stackdriver/tracer.go b/pkg/tracing/stackdriver/tracer.go index 8c06bfda6f..2269e1b884 100644 --- a/pkg/tracing/stackdriver/tracer.go +++ b/pkg/tracing/stackdriver/tracer.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package stackdriver import ( diff --git a/pkg/tracing/stackdriver/tracer_test.go b/pkg/tracing/stackdriver/tracer_test.go index c8fb379ff8..b0d4790475 100644 --- a/pkg/tracing/stackdriver/tracer_test.go +++ b/pkg/tracing/stackdriver/tracer_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + // This file includes unit tests that test only tiny logic in this package, but are here mainly as a showcase on how tracing can // be configured. diff --git a/pkg/tracing/tracing.go b/pkg/tracing/tracing.go index f3031b48a5..9664cbc068 100644 --- a/pkg/tracing/tracing.go +++ b/pkg/tracing/tracing.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package tracing import ( diff --git a/pkg/ui/bucket.go b/pkg/ui/bucket.go index e8fcbe6887..75e8c7b8b6 100644 --- a/pkg/ui/bucket.go +++ b/pkg/ui/bucket.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package ui import ( diff --git a/pkg/ui/query.go b/pkg/ui/query.go index fdada0c82e..37660c05bf 100644 --- a/pkg/ui/query.go +++ b/pkg/ui/query.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package ui import ( diff --git a/pkg/ui/rule.go b/pkg/ui/rule.go index fa068eac9d..22ab0f3ba6 100644 --- a/pkg/ui/rule.go +++ b/pkg/ui/rule.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package ui import ( diff --git a/pkg/ui/ui.go b/pkg/ui/ui.go index b5b55cfbc0..1114b8774e 100644 --- a/pkg/ui/ui.go +++ b/pkg/ui/ui.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package ui import ( diff --git a/pkg/ui/ui_test.go b/pkg/ui/ui_test.go index ce3090465e..a2aa1255dd 100644 --- a/pkg/ui/ui_test.go +++ b/pkg/ui/ui_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package ui import "testing" diff --git a/pkg/verifier/duplicated_compaction.go b/pkg/verifier/duplicated_compaction.go index 0909e6e8f9..c7cd94b4a7 100644 --- a/pkg/verifier/duplicated_compaction.go +++ b/pkg/verifier/duplicated_compaction.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package verifier import ( diff --git a/pkg/verifier/duplicated_compaction_test.go b/pkg/verifier/duplicated_compaction_test.go index dc152a6884..a7d30fc743 100644 --- a/pkg/verifier/duplicated_compaction_test.go +++ b/pkg/verifier/duplicated_compaction_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package verifier import ( diff --git a/pkg/verifier/index_issue.go b/pkg/verifier/index_issue.go index 6f31b63c17..828be21444 100644 --- a/pkg/verifier/index_issue.go +++ b/pkg/verifier/index_issue.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package verifier import ( diff --git a/pkg/verifier/overlapped_blocks.go b/pkg/verifier/overlapped_blocks.go index 1835aa5b7d..61e291ac78 100644 --- a/pkg/verifier/overlapped_blocks.go +++ b/pkg/verifier/overlapped_blocks.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package verifier import ( diff --git a/pkg/verifier/safe_delete.go b/pkg/verifier/safe_delete.go index 6e0f0ad191..b6c882cbc9 100644 --- a/pkg/verifier/safe_delete.go +++ b/pkg/verifier/safe_delete.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package verifier import ( diff --git a/pkg/verifier/verify.go b/pkg/verifier/verify.go index 1bddfe6d98..06f54f97d3 100644 --- a/pkg/verifier/verify.go +++ b/pkg/verifier/verify.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package verifier import ( diff --git a/scripts/build-check-comments.sh b/scripts/build-check-comments.sh index 9648d44de5..d02260445f 100755 --- a/scripts/build-check-comments.sh +++ b/scripts/build-check-comments.sh @@ -85,7 +85,10 @@ fi function check_comments { # no bombing out on errors with grep set +e - grep -Przo --color --include \*.go --exclude \*.pb.go --exclude bindata.go --exclude-dir vendor '\n.*\s+//(\s{0,3}[^\s][^\n]+[^.?!:]{2}|[^\s].*)\n[ \t]*[^/\s].*\n' ./ + + # This is quite mad but don't fear the https://regex101.com/ helps a lot. + grep -Przo --color --include \*.go --exclude \*.pb.go --exclude bindata.go --exclude-dir vendor \ + '\n.*\s+//(\s{0,3}[^\s^+][^\n]+[^.?!:]{2}|[^\s].*)\n[ \t]*[^/\s].*\n' ./ res=$? set -e diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 549db5de61..e48be1e7a8 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package main import ( diff --git a/scripts/copyright/copyright.go b/scripts/copyright/copyright.go new file mode 100644 index 0000000000..7d3dcbdd76 --- /dev/null +++ b/scripts/copyright/copyright.go @@ -0,0 +1,71 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package main + +import ( + "bytes" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" +) + +var ( + // license compatible for Go and Proto files. + license = []byte(`// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +`) +) + +func applyLicenseToProtoAndGo() error { + return filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Filter out stuff that does not need copyright. + if info.IsDir() { + switch path { + case "vendor": + return filepath.SkipDir + } + return nil + } + if strings.HasSuffix(path, ".pb.go") { + return nil + } + if (filepath.Ext(path) != ".proto" && filepath.Ext(path) != ".go") || + // We copied this file and we want maintain its license (MIT). + path == "pkg/testutil/testutil.go" || + // Generated file. + path == "pkg/ui/bindata.go" { + return nil + } + + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + if !strings.HasPrefix(string(b), string(license)) { + log.Println("file", path, "is missing Copyright header. Adding.") + + var bb bytes.Buffer + _, _ = bb.Write(license) + _, _ = bb.Write(b) + if err = ioutil.WriteFile(path, bb.Bytes(), 0666); err != nil { + return err + } + } + return nil + }) +} + +func main() { + if err := applyLicenseToProtoAndGo(); err != nil { + log.Fatal(err) + } +} diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 720423b458..55db7952f2 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package e2e_test import ( diff --git a/test/e2e/receive_test.go b/test/e2e/receive_test.go index be2e1118ef..1ec20416c9 100644 --- a/test/e2e/receive_test.go +++ b/test/e2e/receive_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package e2e_test import ( diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index e5409cfa51..81f08bd683 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package e2e_test import ( diff --git a/test/e2e/spinup_test.go b/test/e2e/spinup_test.go index 06b3bd3dbf..46e4ae2295 100644 --- a/test/e2e/spinup_test.go +++ b/test/e2e/spinup_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package e2e_test import ( diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index b3df6d1f8f..d30bb184ff 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package e2e_test import ( From 72bac9e9e184fc747b245de7efe73987343b03b2 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 28 Jan 2020 17:11:43 +0000 Subject: [PATCH 213/257] Lower restriction for website scripts. (#2069) `Refused to execute inline script because it violates the following Content Security Policy directive: "script-src thanos.io". Either the 'unsafe-inline' keyword, a hash ('sha256-3qFt4qPvMCWVUpjUxP5X57GBKae6RHYZ0rMjn9WuNF4='), or a nonce ('nonce-...') is required to enable inline execution.` We got kicked by our security. Mobile website is not working well... Signed-off-by: Bartlomiej Plotka --- netlify.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netlify.toml b/netlify.toml index 20f4d31dfe..9d97ed3cc2 100644 --- a/netlify.toml +++ b/netlify.toml @@ -30,7 +30,7 @@ command = "(env && make web HUGO=$(which hugo) WEBSITE_BASE_URL=${DEPLOY_PRIME_U # Force HTTPS only. Strict-Transport-Security = "max-age=31536000; includeSubDomains" # Load scripts only via HTTPS and from allowed domains. - Content-Security-Policy = "default-src https:; script-src thanos.io" + Content-Security-Policy = "default-src https:" # Only send referred when HTTPS is used. Referrer-Policy = "strict-origin-when-cross-origin" # Disable certain magic features, lol. From eb86c8df57dd96405a2e05b264e5084b7968258b Mon Sep 17 00:00:00 2001 From: Theunis Botha Date: Wed, 29 Jan 2020 08:48:35 +0200 Subject: [PATCH 214/257] Added Jurumani- and Tangent Solutions to adopters (#2075) Signed-off-by: Theunis Botha --- website/data/adopters.yml | 6 ++++++ website/static/logos/jurumanisolutions.png | Bin 0 -> 8085 bytes website/static/logos/tangentsolutions.png | Bin 0 -> 24715 bytes 3 files changed, 6 insertions(+) create mode 100644 website/static/logos/jurumanisolutions.png create mode 100644 website/static/logos/tangentsolutions.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index 386c1f7ea4..e7a700e22e 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -78,3 +78,9 @@ adopters: - name: Banzai Cloud url: https://banzaicloud.com/ logo: banzaicloud.png +- name: Jurumani Solutions + url: https://www.jurumani.co.za/ + logo: jurumanisolutions.png +- name: Tangent Solutions + url: https://www.tangentsolutions.co.za/ + logo: tangentsolutions.png diff --git a/website/static/logos/jurumanisolutions.png b/website/static/logos/jurumanisolutions.png new file mode 100644 index 0000000000000000000000000000000000000000..e22890d4086165a7df917a02f3c877ed3d873e7b GIT binary patch literal 8085 zcmY*;1yEc~vo@~5gS$%z1WB+gvN#DAB!S@W9&B+ZI0UyKL4$`7U^lqCyW1|d$dX0F z-S_?ed;fc@>py>Up})Q!_nNbz-$Nl!*xF2++{bh*VV+-l3sk`lEDrJWMn+^qy~I zVJHLL^PMsft#+INhBB~h<<#ZS&>E5m@6EAMcAC~(T}9NW=i~!pQ-pCU>N@(~4o;#Z zj9vcRF6h0h&jGXYA-nvMkd}_4&t-Uqo|7-KruC3rLDMz(h+Y1WO+nYm=kf7T-^uS! zVaP_q$LOGThKp&(R0%7M6`iSyZY=QrWtRN3F-liIB%6 zxEm?TQ}`GHFVw1gfl@m<(u@77>d%d3yORMOb4aH#Yxh~*$4$azsAM2#B&Qj3oJf~DL7=KUH;-X1_LpO~?>1y<= z;k4XF_?2+~muk*3Ms-S+XK(82dvQK;{@jIOEg%Ope|LM72`3O3*y#?|OMdIl`_)ZQ zLCk*W@os<20kY)8QkypzgO#tNUoBtnW}>Y?;?4I)q$bEx#UnXi%yG$=5H3rrpIak@ zTXMfZ(C7P9Dt#}3_e*bXX*Xy|>lw2DZItcp&U5$3T29ZS1`?yW8 zhNIZPS?M-GzicpU3BuYtP;US(jDox8_DiY8srkSZpbD*HbdRd0!6*?Y^C>3DY1XSd z2K#V`DU{^6vFT25`09opw{PRCF+*)aNRWMLrJ!|YAa^IP17@5+}mR-5etjgfln1L4wY9XiauIE(5rd(F8tR>JN z)7iQ-@2l&szal8%PDh^friCA_WghnC)JI=<9AY|u=*D?=#260_;eIlQRC7iU7K=!f zXnqP_#i4U8mG%+8R9fz5Fxr+pPMK2%aa>JR39z}UT+mm-ho08^c?CcJQD;@|bxW}@ z5WI66mM@kn|B7b9*0uEKlxdPLCA|i#P*u`Zh4jyej@O&TD>vlczYhXyhZkaSCl>01 zNq5~GTbGNFd|!1&sau?+k}rfdnL^h)o&gDGFx-+lO{f3nQ&n+vG1jjwOUf2G{AOeI z)nmd%4xaaL4-|e-_`CyrpjRw9l&HChY`40><|T#|3hMtHSNet7kY!DH6grSmH_pI6 zY)@#qBC$#-#C;aknu(S1?I{&!(t_1UC5PYR3;?TYX6sp}YAO%MLxhW8(6(36q>NaL zLfjqqFvW0hl)JNAa6|4hx1(2Fw3c$*ug=Hs7pOTt@^EN97T;HxMaN6q=2kxR5VsL_ zSswbtz43DDwpOzH;>!L=mg^LV!#&uVvXHm=Y%^be8ZGVaK*!JXpvwI1qZMJK`<+zY z*SN&}l&2k$Kn&zWJ}-rljS>0@smH>3UrxpO9#+w^OnUAEI}^-;M|3d1r>)~&*+^CG z1sT!fO%1!%px@VaQ#{71{E0Y&nTH0@S}?hzi^!tPP&h^p*E>7R$OyyxA^lpfse5t( z>Miakbvne{w;p&OhR~`t(-SsorewstFy> zO5*m?B&XEB^pSMs;i39SsM3wfFH@GI^=k{Rt#Mq|Io?kb!Mh?h!JQ8t0k=0J<(k1P z`1rk%ZAjrd&~h~`YEc6*+>1^Vik2YfZb@v(viD4F)TY&p#+*h;b#X0!z0*6mp{Y)zh)zkeQcQWVob1kmU^;hPKThypT(1&rJjz8lN=rRkM{>B zShR8D$nAn!L-kJ!?26l-1r9?AKo?wv+*TadJ9cOkn@$vM2MIr?Cip>^t|g~8QcULD z2JfTW59WR;rwF6j=1xKE_H38veLwy&Nj~G4c0($}zU1k{<$fm)k3W^$o6KDCQ%QEF z1DT}OP#8@<@t%0EN++YYMfq2xDm>4}8(BfFL}BIF&fU7J0j3!Q2II6HxN@fTBgr7Z zU}K`cA~865J}$^fhXz);pV~EWOL9Lv+M4(fSdn?De?DztOGu{I}xzcnvykv)n(!qgB9p3jlxj zWR>nrQwg~aLi;-eT?dhL%FwQX;ON)fC7`5L|Q13gMc$v$&2*U9LMc*9PpqAUeEafY;~`_z}7gUJso0nC2b) zJ(IC}a?uZ#f5+ISPPR@iom%|ld~SeAyVeuH7U zzDa2cR`KH&A(v^S1=<9frT7l62o_#lCpT@zp$wrRP~F|tTF^iZ!=Lq`{L$rgNEP4B zF2z-I`>`QA`i?!pV$XTd=E!;)Bk_;BKYvMo6Myy~zH6>yIaa$^THopa6}jUwc7Bko z%<&l@n<{FhypWURG72;b!WB!uCie5-@+E;#MuEADcd|wE(=XKRd8jNk>AKf~DO-;L z!u$HP+B`0;7L-RXV%Z_w#nJpwqVn79?FTlGO5DcXiXT|iNE1mcaKBt4OS^8$s|Z~v z1$f|6p**3-iXHYA?yh{CHOS7&KM#`a-n8^6X$2!w^I(bSjU%m!%cM2WC)VMNYel|T?)8E zh?4|0T__k!mqRYsY^1EBwF!IPP@NZ8&C>I(svR#vLqkazDO_7;1f(3M=}ZS^h(G@> zk`hmHY#sfARSU@GwDMSOhxr)6$0B8?XRG+$3=oe9`dF7C^N7OTz5o@sD#!(IRsgA+ z^n%gP^ORqoWv#A^ys|3E{iG>H%RRjX2o8@{+w!D|<`DJY9Itk(?-eMH$JXPk7rE9i zlRGv7Z9}|m$YR@as9$V=r~u4JQ)+CR{BXrDXL}JLi)pElE|4Jo+Yj`BGm}4d+B;pP z{Lv?XGT5oXTC=fK@k=I^(N_J^Y*wk|Lx9q0*Q$v|=U0ZP1 za{)=;r1hyKV_(=dIxc>djDP9&Acfj#AT}KUPu|-%5eHE{3~7X);p~o|To485?b2Hv zu5v9=Us@h7Af#IQlg{3yGp96og5s}Yr zgU$6|J|tWinynGD1`;D=_%a1SGNBx8$aPFNt$&*u@$kP*P43^OCeBj=f&3olP~r4H%XQZ-9J8z)frz#KYHi>pKHD zSFtloqkouWw6XkzPypueDJ14q_VUMQ<-pLXT;nHj@dKgXRBl@R^~hfm$*#;UCs$x) zO%8c1@XquOHZJc)>tta*5=WY~O~Fwh`n3bW-jjaHG-;iqO~y>=i!QlPQ<=cqyG+4g zxA7Tda{w0H-ShImfm@_@0ULf^0!gOEBqm5+i3skwkOQ~yNvi^~-5~LdZ0!9Ky}I!$2iYw}?cX6&0}??yWAc9lYUTh-raKlDUbZV3X^l`(x+kSFXVvSt13(guxHSCaIXvJ9mB+I!mudN^aQ=;TbWE#W!ov zI}q-JwC=kQ)h&n5V${b1`+;=FXk8|jOs)ZnW8hH2HmK&1u-GsFqv2rA32{jze0E(^gFVp_CF#z3{bVdRuO`-fC`8 z1d<7EPzb~dFWx9zKq`_NO`VY}%?1rcjSMREL-}#+(#_<^QH_2P*eLN8bbp9BN4l!4 zb_%hRq&Jl%QRut2Nfw^~Egji(yQ8;twF?^d255J5D(eNM1kjc}VHZ}ib#Gd!Tey4Q z&=5L$0mOQ$x3R)*|*mn?y85#-J@Z zCP2-yFGLZa!v<6iKOwt*iSftStd!b;#XS+J(&$>1Dq@$CpmNqW=AOZk7sopu^WPr^ z-Hf%x$(xZ=d-8s8LC_Sxqm+q1yrSg~jxaf;`MrGc6nYaOKA{RM;lYm^8C1c<%+M4DMCc zK%JH>a6|JI@~r9jL7KG~mS&Vt_(=4`cY?!uJ?+VURq|%Id0x6FXX>B`cpzFk1H)=) zgj?5^*ObSmJJR}p_#;=JGkx~!MUjk!l`(2HaqO*Mia9fNOEC0HBr4a0VVPpaihz}Y zsTEx5SaT13x0P=DuNVdRC}>OQ$x5k8=u_+}&I56yvl39txpWn@uVM77Qd_+HOLRf+ zk>3P3z%dkU*F!i(sEtg{0VQJFhAko*;{FK^Lw2tSF!qfQ?{k4|;inn?!uSH5{~x!a zDyWmBoPy+Y_=&uA<{fzMPV#&8(_cjJ)+T;{QKvAq0LT9dt`x@YS*l1J(OR0LJ%Yb? zH_<6jrq&@3V2fS2D(GPr8TVqQ3?Fa z0H^<{N+8M}EN#>J`Lkhayy@9fQr@xoU7d*vB@PD%(uh6?_w(7Uy$B!?Y|ROk(Mi5o ze>n7wRoL9{o>~>5wIlsQJyCh!)@El8m8~ed|6*zNtLki(SzT!$tRjC0aQ8jUCgTm> zhC!=Zlns@{agigj{XA%@VeH4dck)tud}yE3W52qp*Y7*RU#z+X&5i`C2zuj6n}4D1 z-+1H0*%!l$d|0@H&QkmxP{Zx3$RArFTDx4zA>rNwWMWUDi2s-}*- zdPS8mGR&=1ofIr~ay>l@_OKHJXC-dlqZb;#DHnxgzF1YBtw&hIfH#Z$Y#_FUUnqcR zO*q+j{;%xvUeifT(-uoz-h_N!i%~n3ZlF)8sAeaumsHJ0PJ46%bAC-QtZusD-;a)T zf%a=Nj+y)f17$jD8@>a%!p6h83a;XTR1%sOzD|<)$8;RUtofHTj2Vg?bPk+$sEwFk zItn_^i>v)wUo52k-ufgWb!<6#t}Usb?T^WilA27SH-9Wkt+_pUk7Rj)xkE@etal_w(9gsqZE8HL`Q!y3$}W&dad37%868>C|CflKE_Sc~b!?Na?5+M%+Webf}l41j>V$Jo2JK4;9H_ z^3NY;G8Ize0UjM{pkwy=iuHodc~|qT-{d}0 z5kWrvxfRE(VjuS(xb2hMjNiA#hjhs6`9s&Re7giLw}6@plr(BAxaKBb{li(@=+ET?3dk~y%7w1zD-C@1VFe% z0yahls(m&X{)o8OOn$7{Z8>9hc~I}d%j0`gsyFz1$IW4gqDifDX5k6M2Lq$LD^vV7 z=r8**K8hmFO&F6uEneOwiZ@<#P2}W=>}DRxFugnpWP|L?iv$wy8RXn`!H)7sAQHJr zp5WIYu1pFj;>GjMdbE5N>-0SU59jv5YQY4b7h!eut#6Ubtovv0+WQ(HAYLllAQ#@Q z=FR3aLf?eIf~m}zYmzT5K9PLh6LWV@lO|LJiD47`yn`uGOv}DiYeYbYyxO8@g*SsV zI4pn3xO?*bPDAX)JLRN#TAe)`XIR+jIS00`q<>t>;N^SX{8Od`A}ZsDM_aH{6&_F( z-V#^vPGjMA_{>wyL!?IGT*tID-FNU{Q;YCb^gFM}sj;Ax+W6qpD%ty<-LOZA@w{ol z4n292{=+WPN_T%xSz!MQw7&E|mHQ3yQeq&&=-2%0gu3#ysCoFFsiLqv&Nm9rb_q*&F)JIb{kY7juYo-gJf?<$ma%5IV^ ziAFq70bNf$WA?tVU+qva}6#jxoc4e-G1rM;r)*|Xh z2Ip(m<<7Y5ha_#UZw)>Hz8dAr`auK6G2C%KH+iKqL0>e-FA z9|~&Phu^+ExSw1=FkiR5h3W-B#l;Y2TB7z%BZ6*;hiUg;$|G)Ir#xsm>=slZ*PmU1 zP=u3?PQvaeI_5zVtbCiUyk~EYWa947p3pCn1 z6re*;EM_R}uO<8p?Q+J2{fFp+Y`brF7XCEV>l=x@NDDKC)0C_q<{D5voPIN^++o?= zOKs2%zD^r`{M>Z1q`j=H5CT!JFGVp$5u9uJRc%f6?uqWxbxlQKA2QfRH)D=>y6BONMemj#hybDk+ z4+Uw)D&eQ!S)B#e#3tAUk+qSDm`_HvxJO_P5Ep3k$ZQY(6Z|C2JL=aS=WQLH&jtU% z{j#FPU!`<^bK!U^rzl$jmLAyS+d)pMbu;)YX@90VR zySrI_BA>ZWS$gXwt$Nmqa>5D`;L0s9V{v;R+T^*hqb~8Cn9Rd{epEs0gUP{d(ZQk; zGha9qU3<|V`b~6t0Vcn!aF=?!{Up+)_0yh4)z!SbphvGNvsw8wnLy!fbzqXgL3y{o z5BRw8OYVOcav4C%6Ubdk`H+dDt|C1&r}1S)|8#fKq*pznAbT+~2_x=&y0#=kS`3lZ zzI!~g!8$qL!bRMuFrf5dn$i6GPx8uh7kNy!r$9{^q#a}Pn}L2m>6q9;UB{NK#;2|k zOIy64l2z5_`Ga1q0>_Agb$0>c`L>6m3k#FXhNQMQ&;Yu4m&K#MDUk zXomB39RY~=!Z?9;7ibr$=f@)3fZg&srsirA>}t~!+Lv!DgZVVOu#;)vWeIkGQ)2>v z2)08{af*3iYSuxxu>YZ3qT{$ME~{VJK)_P*Q&tr3&1C+|2iukk3wQu$pu2Hv^CaPj zS=Mj5XL~PK7%q!F2^SmfLg%!-7^YKaN!P9hy7-E0bKXmz!Z>1VHeNI9%8nC8uN&_% ze*d$=3ClV%ulrGzE>$_N1YK&6i_|w}M~vz|t9d3y@FcUSb5$4%2=5HYuW<5Dl($ed6w{CP#9sMBb zn4fr6zEXMP5x|=?W&c9yCrpGgw8y>>%&O8Z{o%W1dtLhdSzD56wdf0|r%ub2m`_`g zU+izQfA^p?A}@)+)!sS>9udgz`zmlH(~s$TdOA|TT=hCK4d+@yQu3847Q zZ9)EIXrESkab3^mZ`t4C&A9sCX@qn>oaeemN=L72(Gh#%#hXXOidb~Xy?68tik1v} zLndbJK_1d;8wndd?%!>Q)oEl{w)^t7J|KsCn{e;FcZYRHPgo;2-pQV4rTmemAuE`H2 z#q{^~efutR14Ao}0x?t9-4?uiB18r{h*DFNf)S`XKxCm~bkrb%Ngm-cZ`>Kvs^)Wh zGV~)dLU6j05jC0(bR#@fvlMhgmkgbsBWejMAO|@Qxw$Ta%r1eP#iDQx3!UIJYW zqHcgd--4q?Po-t+xY4SVX#dk9k#%w}N4kQ0`$-LAR_YAv)hS)IPv=%rFOi(N9tG*#>hKh^vDlL-5++1U-OP+&^&UT`H`k~ z+(>tJ%r^$>HE?Z4O0q4QvjnAT=O*vSzlJ4fW=Omf`FwVJVP5(OqA{-k!ux2DMWkPP zgFxG@fn8t3sUc{um~SH?XB*c~^a?~kXb*#oAP~q=TUf?wuWA(pmP) z4Zf=cbFarwQH(O()=iA+D5gUf>X%*OZ5SHzoA0SvT?D7%$y^hWLaEf8&tne+yY$If zl7ue~#Iu>hjM5eKJz24E`4iTQX(~-atN$|LCl}R~UcbvSeLb2rO42k> z-G*vlIC^X!Hy)0Iv?lDijnssYvtV=xG>YUQnipO8Z|WnBk+&xG-(2Jsz6zb9xkdwd zWZo?Xd-}%ct!NAMZ!edtyj&boUs`CKC5=^T?S{9hkc9TT7+f4&)^p5Y$pG$>`#c#Lo0eevwuQ%L+BYEv&Zzudx~Gk%emY8R%B&oA+h>cbFAK4yJFSvvi#C^omQJ84CBpyLos_X zJiAv}GJ7>MDCxG8MkBc?w&#LplS>)^lEVIHF$~2bRr+wJ#s?M#`R8 zo%kP??dj1YDeLHkDW~a&xzye%1O=+`$@WignQJ-p+|YXMlIfED?NNI3s9G^mH$|5z z*}KnvQFzh6&%du&lu}es)W~GjWIflyq`6e0RMlA3xc}pak3}EHpDX04%T35H=Pl<= zb@(#6x2)3fniRnPj<_NS38ktrwYR19%n8L@n}d{KAb z-eFULtIXvNzgWK>`EECBMb;BJ!aOp0WwC*ENTSy}-!gAZ&y&-$`EJZkd{TnZ!y%b$ z4VgTizS;Nf-wlKJf*ihuf8NW_tIO;D{bss)U2)xeJ#OaJhpVq7UfNu_`>E%>7Rh{R z^g7+E%*(#>xF}OCQ;cW*vXHEsY&xQH0^j&Lq+;J?gHz*=#+_G-8jEVz=H%yWoIYG5 zG`?>fGVKX`=@dJ!6)kjj`z-uy6jX*5A@hVJU~XdUz()t+4+J|?ud7VE|Ec}@`(|=M z^G)`fX%8zOE_GE3dcFVkq*8EyIcGS1*qbrdEb8a@Cn}d8E=fI6^kT7bv53dlQ&I6q z?oqjHg^73bDr9`rrmnSIZMohj<6-lGh|EVC_!;pk z%Scu!2KaT%AOGHoTgYFaTa4h$k$vxg3M(A$-i`_y9Dhsc+2&1V$u}Iy zBHOGXBB``t-F)4iFxgSX(fCO%drAIV+xn6w;TgZu+Z4it-(7hvz0~|jto%uvF@y1J zp2|y(-1`^(<<(W%ve3Wa|g2JY07uhMK9>XbdYE6j^lKw~BPNRaCoZxQQ>uv1{;QS3Q~U8l5%m zQf%bf%*1a8Q>3iTe7(!z<9r?s_JqVLwrS36_UK9Rnb0MkxWK{A=QGdVmScOHZ;yC< z_E`Bbcj=`aV^&4>k8H*OhObj^8Rp?{fA9U#nU>A|C_dd9)1cK{+^N>o=3X{1{89C~ zRf*NE)uEN*`sdktmx+Ub(RH4Pwm(;Y5vT3i_D?hhI;O43tWAg>ME4_uqq_a;RS6X< z2U$n{4wbk3gMRlzK0{8Zgcutpn!gbFKC4?W3h=`r>Tbe!gIgj`}!WUx+o(YGLk z(hmE+vQIO=&(EMTbw4=*_l;^y;69%xpRA8&?sl$jo@MTlovU4o=?9I`@zuHTW4&ns zrWaS$Y{Cq8tOmTNRvwN$_1qpw&dWar8oa7@`gF{+zx73HZdJ3rZ}WP1dwqn*{tC~_Ys0g< z?O|ISXXC_|%IdT2JK=%*@AmiRIcB5oh@A`%wSEl|Td!<0ZFAba@%!dit=Tx@b-UA< z$&Hz4A3{jM&ghKZK4omo$Z7gs8j}`9+Vixb#|)3xUMguQD=%EXbFy+=rZsYVMD!)+ zY0>FW6(@&m)B$mAVt6!Xv@m-+`)Q~DgV!gkgtZIR(|z52{2c?KSL7Ub;YYu_LS{nv zHj3w&Ous+47%hJ0^|xlv@>AK-(+f|Zo{>Xxc8=YyT$%>*&5LgO7A7E2xCjV@jsb!G zo&i5sK%je6 zCurINZ-q_-nK49zN)ppTn(w5_qCD%+{>Ba+=tR7Vu)i5^_FVHK40tlGtU*-CB5SmX zKW=PDSGP7AY@NwcdXNdY3{)u-dPgaa>RO3B!B)r0}Tr+{Dz2-xSp(4@+_JuI>WPF%DwRT-2k|yb1 zK_Ru9Jz7OMH?3CjaW>82veQ({I$K&r&s_m6y}SVa^>Z@k zK1IWLl|+sneohnicbrj)BI3qXekEx1Qv}wB(vQ|Yb??I=wUTJuwK!Oiy4W$9l6E0; z!jB{)Kq+E9@6TArjs4=R;|xykQM@c@!eWFw55bMjQYW*kKb6I@4e=g3OwMDpfqm8G z>9E+hWGMDPZ?I;0wzVZCb|v<_EGxe$x07EQ zkkc5GA(r%4Yq26VyL`eHH~u@@0RD47*e^1n;hMSJ6wd*}_>8?U&n&*Y+zk`S#z^3Z zQwI6w9>ef8ny~FF0NV^u31$)eIngnwR+uXl&d`{4xf;oOA|ICx~?0Fi4XE)Rl9(hJ!fkK_=`wqvNaO$g^tg$SlTLopBf`Y;CGG3l_ z=gJv{k25Xo({2R@$g=!tJ%f>yVl{<3{~Z$uCf5y>ACEMX#fb+~Int%Frz?ErJ789_ zD11LPOz3a%<|K1gmri&chlGWdkY@qT>U_mn%(PF1?Ps+EM?w{${RU60CVZ5!iPBeh z&fweo=UJ0J1=q;<*cc`5v}5cruN#eZGtZBZh>uimP z5N8W(q}x=wtX=p7)_A4CwJiR74hs^%l$J`hw!gm~9x*I0uHSd2dfQ?zH%%AHp};L$ zf+3uUCy2bOq+hGsrvfC<_)h{8vf#*eD0hN&n!!~=Otjm_+1CfdJY|p(vP!nphdk7N zm6!F?eHmWFT0@fbQ~xm{GtFX09MOw#Cc3fY{~=(JkU$AWcY*}cGANL%jy! zbOUfnnCjmq89XH_G6_M-`%Y?SE=l8{><8EbKE`7$~sC^Tr4*}0Qz)}HB$%soxz z@!~S9CX~kxl2%-gb!uxv*yOCGKt{{(>}3!3_R?OejR6zsW_Is)AzP`sUwZhPM}I(wtb*@-iI5 z3**21o!m8n8q9mVd1xi<|Boc!$8OmDcOTc0#Q3H~auNa=D81F6N^qyuDp@DzH{;T- z?U3PmeL4P7*}17Vm2l#HjpwIPR~-mC#duv(i|DqNDG6MVq6vLIfmy>)016vVrKmFJ z((WF8mkXS@ZHaTT;@uOAvJ_feaL4J{BxrtnYe>@fb2m-sdqARft^jCfeNEURBmif9 z{t+cLh&RmnAs3Zq2jTHSsFs1(mg+FTF+TJV`(yq5l*$DOSaTgCiXYz&96t28sV$sc zMOlG`u3uO5J`w4 z+==Z`cEFeZ4#H&|u^iAFl~k=C6+56FG^B>UK#(uX8qBwmjxDr;;Ne-IdKG$$D5_a% zUPfbt679k)scoFs>&|WCdIE;B4ZZxNV_Or0ZtFWeeq z(}eytFY-|SU9Fty@!L+zQj;uQbBcMgQGaeth9Ek%G^|3eM5F%w?f?iE)S3nd1-eQ( zR>g>{jPy$0Da&w#wL)_^j@Jb+>X<$J9EQ`GNItZ^$zKY(2AcnA#x~H}j&229foS;; zi|{mAlM0V(`SR1l4=rVp%O3TNt~0bk6^Vd=bj+! zb5b*>_3pMWZaKRNOWQ<+bSjKD7E#PEt)p`D?`BcgLSF~X>!`gXACK5VIbYo5TG_8| zxpn|e@3!MKy0}qj7JsY_DD&?;cJ4Mid>Rf=ra8$rS@VAG?louC-{H%9RHr?wh`6W6 zYmgT|x|Ehd{bYLJI$qv+PeLKmdNN5cdf9};?cBkrmHvJgTm`0-V1wzN`*nhak@FF? zQ)kxCtUq(yKg+cUVLg~5^qttEf(PXt;p}VKe@vNNRoH!xI|LfI7jmuK)ax~fF5)cB zHwmx1_A>NG&*cx7GH-qHXQG1EXgdHjst{OB;Rq4c**!c0^#oIge$yvuJ6vP8@+95p z?>{ixRDQW&#6EZ>5si)VVS#5&e|xq<9f#HP|teEY5dU`R%iQM<~CB;?!$b5 zqzS{CACn)2!cX=wAu-Xf2&Mx#@n6Z-#R&o`ZtY`(Y>91Nu?iPLMvDnLaw$i`&JDm- z%l9Z`C@Ce`ERxSdQMx!i=zh}Xo*3w9O!OJQk-qlQ4IH=0aShG5S)*0q`{up)0GGJ> zFrEVl-ndm?8>UFUhCq~?>DQP8TC*^ermMi)Hkz?#eIEqc!10a|ODi!3iwOZDH9vkD zRHQ|by{Pmpe__Xj9S$B?o_u@LT09|M>OO@39Q-YetxjbMar#$B%ffvB_|L~ONZFdb z^15>$Cvro_;0ph-i>RB=56iE{Oz(Ag$N2LUIWsfMm_*yD9^43KATsrG51M}Y_AL`z z^J8KkG`Q|r&5qA@bMqlTHF#^C3xi?n1QaX(f@3ukV_A?H7GQm07JC1$7`|!T#=9J` zOLqGA-1t5kzBRKNPK|Mw@`|4ImR4H!$j$ol&{gU=59k;Fx3y}u$jjv~y%0h(^6^P5 z$kS#GrTcx7Y&Lg;4oOB{E8)tTsz<0>7Lj6L7z@^MOrUnOoezC z*R`*;ThxZ|q7u5Fr2o-wlwVnGV`wXyg~9m0UO3<}du^s~FM$#9wVL>%f1L?i?ePX6 z%9@AOf9FpEp^0s}Ey3hU@?g{(+M$qWa4Lf9^Eg>ZY3GCf+%oQ@_@-Z=w~~duIqTc* z^-&caJ{t~u)(I8W+tClC?QX`cMU_cP8%$hu9;0@O_8+4{DDwOM;m<5YUcb0Cp-ym) z6}+U*%RLlFh?x?BEfn4%Js`niH5c@0q-`p1?-AiIE7jhtN1}TepvCT ze_&e$+wEiXT2h+E?TWMgyF&Y7^IGC%!54AUE4>RxAJXM5LQKXV}SSblJ zJ|kqD{pBzO#aKm(xTIz;-3xA7zgjWU{9OB^y!a^p6XKNc0?)x$!jSjra4S;ag+GjY zXPq!Bj#+ADA>AOUka&T@rvfd*m@G875;INFKbQf#&S4840dFw-%XZtZk*QtiIk8UL zvu&|0lk@xyvZpv^K~|ySH-I<4+NwWXrL%TQXu^qIR_mTH^#nL7A?+6#I#)vIdk)1D z<_*zx_==8babB!-0?$Z+2RKk~cWN5!xn>4(miq%LyjjRRkRXIsCTy_NZso5ZEa^!h zoc4A~4p%L$ophpSaCk{v21n*=bgO9m_He#dP%4b1t3khJW=N#nIWZ!NV#19?1B;>sLAi5)QbHmvh zWPag7uW7d`6Lt<|E&(@hEWbZC@(fa?rf?*90EHS%z(_%)u(*->PB0^}isjWxl@kct z20p8(7QJK#SQlsND|SHMj@HOp+ZepWBlG9RKiJt>6BIEN@b+=XICHNt%wo{B1R|O5 z>YQ!ss$M0k4D8!*KQ8|En=sTYigx`^?KCsCo4asUq{9Mi1j2T#F1qg#S!w(o@BWf#=R!#0Ib3PXpy|fet_ho32jvMLQ6{s#5IXpiNe<%4`PG;zPLe*nE*Zd#IFN^qN~y zzAlo6zQ1BhSp5nHQbR1Xw@Tgn#a-Ya0%K8ia=|X91|6vWG{$io_0kQ3*yI3h>A%G3 zy=|`sX@KX3N9S{I?T%41^#B({vZAk`ZT0#a8OOi0$KJOt$#S*^3gYHnWt7HxRq8bK zfEC{g5OxC~44$nIclukmT~JZ2-S#Q_Cqm!_=*#lvhK_m)>LQ#d`n*}7iEr)h_gogg z#GlwHlXTl6=6L;=ZQvKHn+M?#PO6DO*ey$r0(}t>3p|q8SzF2JcIwv*+@>bS7JtWa z+p5+n5@j6-Jz+yb@T-agAC!@>+JUZ`OpqPve$V^^?=%hx=JbOIE~TMwTmE$RjBlvo zl}rH2th2p}Lfm~jE2^`O3jk2v_b30wx5d?$(F%mcc~_Hf#c$Osx(CMt2x3jmAG9S< zEaO%PZS95E&;u$|T#uU?Ta!Vnp?#_G4X7d9$$zT5dKH^6#8ClQNn~V<@Xcp;0Ce`2 z96GuGoCknu8o0WSMNh(tQ!T=C>~pm}HU)e9rX#oIL5;OjA;49dGIJYDs|( z3XSB0li!bXaX-3!Co7DG`1qW}NHhf&trgC;Qsgj;i?2Xs-_{qPYhVQU$QO~|`-~td z9|d>%ShtGxP_ftHy++T)w#Bur5P4Z*)%y`h6?7@M$^CaYj?`1O69UJAO#^t=zJP%h0VE9+@8Nq5pJ|Xge%SF%7CH2O6tVJ9-g6i- z*Z(n-CXdX{^~3_GJsi2wtr#Od{zXOt5t&2{^^{31(f=ZnJNdo`$O|euShl)E2`=Xz z3eS3FI?sG^wfiQrKJPmT3*|~^HJKekJ3*Fx5KFTv;&Rfk*n0Y=3kR)$0zge1Ya`+; z%I*}2y=g>*wFdm3pc*q>v5VF^N{=a>?JY)arwCt$J$ql0JyXf)EtaORt?9jV*PE8EF<^2J_|ft z#ajHn0d3stDy`R9aU*>Mw~r?grlFnlJ@|@i11T#6*`UVq2zOLWT@ZaBKX*`Z`qpED zb0nL#F!*UrCXFwkCgq0BfjI*Bw@g9?3sUDInJ`NzB6825GGD+tQe>e`gK(eG{@6tP zlP~eVzjx)O_RTfeNI$z?=8)XRzIgxwbHEAeo-3TE{p+1gVgve;d#+ z7;v`1)>UlzKbf`=ZbH)vp8(j@2YRJ<+`-fjuXsAs%yqyCX9rxjKxnjj_qi_}UiVPu zXH^iQDuC88 z@n?NMXe_>g)+dk@Ra8xxl($hMJnr@7ZUu7|RrV$G=)Dli$IK?+H5Xj&+bVRm4O1{k zhQP1e+ViDQC^Q7=g+CuGV4`l zt?5W+f?Q%}gckE7<(7Q+h~xl%9~9AYeIqE9I$N8Wqg)g;AD(=&iM#En2A)aDGnrky zEtqvV_@?Phg|1}p(LMjYcifS}+MrO%gZl1^+pHpZ*VVFBsY*3*SXd2IG4hh(4&J@2 zcvo%I8ZE*{OaDpm`L!~TY;o!O-?#L6@{12o3F|s+M_D&sYc2j|1R4eKKeD8K+L-uJ zwr(!l=0l&S755#JZHNfi+yxMQYAF5Pnp5o_p%yY<@d*Vdc%d3PDL8IgWOaCK4%d|2{Ag%M<(cX#uQIU#n)EaQSzKk zv}@`umB5w8meg*d(BrshB7%6XcfDwmer%US%;?qb2YC+_?ji2!M_ECh@?9_zbH1J4 z8>wE93Ghj2BFJWwZ;GkIJO?_;R{)unYK&?#0fAVdr@ z!1Kv@OzZR`_p83FN~<&6G)_h5Y`T!&JH=v`Lts}xioA@+&GHdm zu~SOu8alu09V8<1!mT0|9Hp2N4@xctTn)@tP-VepJUyPiaB$P*eR(q>0-%5u1^Xmd zx;4l0bKz6y7lMQE3tj1iJyW2{O<_UGxeyVT)NviQB>)}KyS=6539SuOEt2#UHEc$% z9}(hrJY+T#-Q2jzUa<5a_usV>Ka>{HN~hyHr~+*4 zKcEcl?h5j!vZt_mcd@VEg<;h}o&li#_Nq7OPVCJ@3=8rl*7WLVd$SV}Awc|q0KDJH z5K~ASgZT5V;ztSU7N}2x-o}o$jr4Ppu|ItH!0$5Zs@adpCS8cr+9v0~uPo*!3Cd-$ z^k>fq+d0)l#G+3H*|~U11hyOvHB$djgiFv<)Ke=_zk!R_DpG}h9j(a+9lt6Issx=< z`+Xx#_7!Moz`nkr&~+gqVKuMGv-%%Yb{tv(p4qXI7V$C4{kJW+r)E`I;>Fj=!AH4< zIr*}fBcM&CoJHQka`y6_!z1hT=2FTWT#uu$ZyjFZwhJ6kZ#Y#{!_PBi ztu;`q6uqMgE%xQp?p(C>ivE3#u2~6pc7gyp8l3sEcSlWn>qn6%`hv=Lojb0^n_*6c zX8ODmm?a6Iw&|0$1_Toq4nC5gcjN{K!?|EV0;uBdCz&gRkEv-};v}0Hr>o!8a9wws zix(pxzH_p$<}Kb$-`Jby)iP6bC&+;dz*yx+SLzhby;_|8UG5jeZ2$BOyvIuMMz~sAdEZIfC+gNVx&hdYk}m4-g{-llc7VyHfdkS1G=Am?(yWA8gTQeIS!EW#F~|mvzABvi6Mmk1O2lIbmtzGL zbwL7TLu`rq!e}0Rhjza>uXkRewTr{Il>;h@>)EbE=0L~>#0_IZSa=d$=8JOVk2gW^ zK34GAAEoP3HqfSB6fC%e%F`6oumlekOF030(#QCLepO%Y9XuttdC}f{mWy@*YoSfp zQ{PH6rOrdpm(-&KKBCFGMh1;$MtU-W?;UUaidVtikqStXo6!rhcB+gsxl}3fd1!&9 zfMbnGY_ypU`(DGekl6_^$^;<)VDtyt5wNfc$XcH8RyuCV3@l}vGWlgD=mux3(4{dv zZKEg2jqxv$<(laYa>!1xt!alh`I;6a%q+2 z9u~hfCC_0pFSTim^&21QZBFET%crJ@Gg4plW5)gW3gBpP3q((X*E#I>p4n(I#~8BIlY zYtMWFy`V8R7yn#K-UJDCDBSFGBkTZoS*~KFFw;O22I#xWI1$5F+|ksl&$7bbU&6ngd-xjLbH)6gJuhD}Zq zO|0gF0`jclb4#EYXICPm*BA>&VHdxAy9M_3#7*6r*^0zFJ?k|DmYsp%uru6O8k-cK z_X9Kmc_qcRpdy0jB|h;gN{S;QSkHxnts+D*Cc9~D<#T)=yxJXwtj})}B1Q%dr2{~M5`B|bh zFOf1y>mMS{=Isqy@8VuavReltYGy2~Ba{-D2>03;IyZ-kl%BgHE}-d3B#L{}+j!AF zu983`5s;~r;VYj)ZmFXvH#0QTv)l>(lXo^39w#8TeX2AC7cyIG1`VKMfJ5W}ilL0K zh*YpV4;<5MliipgE7mjXcaB~8k~m-mLN#cJmJ|w1A=9(|5@N*@mL4V$f(YcXn!T!f z$m_T`Hrk+c7G&yH-C>0-aK-5=+C=Q`%@;sdJ*6vOw4BPB7_#tk10922qSdhlCL_pb zggES}v15v*xOSTYQxkcpO(F9H5R#wV-m~5q9Cx&KN-2L{eV%RXfm4HSOOGXN>~>|Q z^rUz{vU6sbefQDm;RsrbE%EIMdJolXQYhYZfuJlNCc;8Bh8~KEPE^vOk&Y9_R^PV* zypl~<0?+S${$` zI+j*%Ze+e6A|S;EtGRm(mEL`^3IOIzEKZv>Ume@Q4dmg_`Frp3PZ5h4*(C8lRmMVo z41Tz@hhte(0~k44ovBqkIf=}R+qgrG;uj+H5Z&zoM{&6IYJ^h0MGkqX$Nt^^SOf9wYj@;L)Qt3)ul)E9fr57ctfW}L;+s8Er zyllYLooof)ZkF~=KuU1PG5aKCsNJ{s2W<-UO^8Bf2^Chf5hjqGHX)G(3EinV^m*|6 zJV!k6S9j*r>)XEc?i$b;SD$&#a{nScQ!Ld<5tIhX7H6~hOP!|~DOk}*>nnlsU~Vj@ zjnf|u*EnGOJot!(Mg$K#5BhpW(dUYr4+rh;h`p?%R} zY>7B&iYfw!Tqyh-65EgYh}Ru%@n%m~#bAN_;UPekkN(w~!gDu22kAOPXT9uOQf$k+ zUBcO=Q2u!9<%A<2ppN$hh{{VfDV!xc(EfJtd;nR$E~qlDeVaWS+$cqF;brKo)4RMX zbxu%!3+dL*9~**^PRI4Ro{CTQC~0`USQPT12q!mt;)_r^F4|uydtnkDPk~NY1r~A7 zEB?LDRcEPmjcPM}L7*&1*GTR}OQSAd<_e`1(bEy$3;DG7M$?j3p9U9S#Z0KEejEeYX4n_y4pU$P`Wqh^frcNdhp8n-GPM)NTdVTUq3l2b&Pu49HDKkln{mdHu zg)ADPE5R1FbpfULiD8Xm4{WyhH^LBZ$43=Ro(k^sMkMJ!J_Q!6R@rC=8s%dPHWBoo zb~S9rA>Km+sEoGY3Y&~(0$sk@~AG;)#lHp zDEkd`C^?Jn#*EnL6_t~`pvi*fkNQ8!1G^a_4Y{{F7MygqZn2kl!51TVy<)BF_$mBi z1d;^4P%h7K;#V;58bbT}k?)EE@Wtkur&7VIXr`DAjHW3zfKs}wsqFJCQ!s)5nauFw z{;|?=bo`S+PD0*|9a5XMmp>}>vG#!AtF!I1E-}ov@cY#XiZsCbX=X5Xo?vGOy@ZGK*kS zIx7Op$p3G-@rfj!)@x~1$|(a;w{{*;V6a-sA2d_^0f*Cf-i6W+p>Pj zf>ry=VPw^wV12(LJ2O7M7jQj_FF9UCmOAXJ^()^$=7@cfcGmm|-s{MCE3&$}g6i?M zdUda^0@xWS@sim`KkpiWV2`66(5FZu+{w=+Ma6G5;1#L!TGgHWQ-)1tMUe`6B-io2 ztslkMSK8+Ics4o_Jng+J0>@*W|1~0jHB89ehXPP2bN~$(lm>vGIP8CH6NUob+6Rmd zYoEOFE$Cq=GymLX-^C1k3<^z8mcUrY=N*t<0S&FuB)RBv089USf*t`3e9F-Ws+B}| zQ5A-9;#HL-utjkG3;-)$e>=jMPFL*qh*Ur}{NF!A^~5omo+%0B`GZnG^wzm#Uj6@1 zyg0wDE=&r5q4O0`;JFPrikc75|E~WHNawn({41dMm7#V}LyIjk{$Kjup5LgR*XVlJ zw4z$Wp&@?MsH3c;CTZOz=-CH45@A@$;+{*M{^tSu>U@iLE|pD z{l0M(8G4b?thPR07nTEt60XpebD7?cAFF%9#*Og3auVfvqIZ)kx%x$~#nybs3IaxHMhdXK33f3k^_ zz+`GMB#0{F-nhQH?*I&)#p>yDfaY$(cZi!c@>!-N$A5Iy!Ut`?>x0EFdF!54G|>Bv z4kyW(I&>iak7YFx{GAfH(cZ^P*1x{q*9z=AHUL0I*p(qA7?|(>qdvZ*;DZh$5Uvlc z>I;EBzHFb`y}zz=jw(Q6{J-pg;YgtM7^Gt(LHD1x0RJlp=#N4cTkZ3I zAC5BT5Yif0#m;5Ew064m9Ejct_6BB=nQsEkF8?P%VG$9${FfYkl@W1Pv%eDKe>*NDgtrjNU~(AEQ;$J zpnHTko!V(5Ocf#PMqmNOr%UlpCx+7skSy>kND^iP9Rd_$u*h(2VO?8_Wda#N_lEW6 zNEP-4rx7NWaJ4{DNg#^r_-;bCxQ!Q;ALTB2IQiLheErPn2e1d#S)|A2d_UFv1Pn$| zoUEJ6%}S$!unMz@Bv!rER5cThe2v^*y5St*eDRa~$IhB*pVP+2q|)2;n<|&NbD&Ro z>ijYJF|Pz~()UdHDx6KLkveZCJAymDxJbbGy?Gu){~}MO@6fB2NC>26q23@H0m-^0 z$fPT*Q2wI+CGbFY|3lZjF^CSGhPn3eRS268bU^H*n9+`OKCWYR4QTVeT{6NQR%kaH zCcq8Cp6_*R7avw!_;dSMh9re!`MFO(eKvKhG`N75L(!v{F7p|Euul>}Kd|7Kd8SVH zfUb@D1(n~e@ByrX4d;Ol{<;T*6>}HOz?V93scWKdDk@;`+t&Ao{78%Y{u^N`pJt2q zcd2l$clmi8qu7d+`CeQW0UpT00Q{J@3t$ePYS?!I+P4u{xS_Y`-9FOvaSABuO95 zX%r*BzaYUj=}3qh5pjz|sM2+P!#40QhXT5BlN#-L9H|tg%GUy0i0AM6>+OMxnqtpW z$=}#W6C}(c<=4pWWS59P%R{T*JNT~6NlGPgwO!!i;1!B(NmbzuRZ`*zq-gjo15LL# zhYJZi0V8a{bd^ZJS_1ptFeB6`QMk;_?;9Z}oGirEAosePsfEKR3)c8AW7ET*l0F&i zjDap=9QQ}dOtfL2*$-65`!{$mR+QztBi+a8agq(4dU9=J=^IY3PM};=$K2S|LjdMlsPLUt5rADQCLDTN4oM ztq6cOy6y*Md^|Jy{8-ah@!!ApAxApxFnbOs+?2xF&Cp+M^y;}NLG}3tjTXY=_nvFt#1aLClg+I|5bd6 z(bp8VD91=*9{|xnuToz!6WN1wI+A;4k+X8qeUeu9tgn*a-h~g+_L%#9qW*etuVmFa z?Cj&$oo`o30!Ymv_eN0b8p;lS^Uh*mu&Wkn!x6P ze(Pu2-!~Gh40J9KJ!gMS?RMSqYhKp6a(wc=cC*?o#bTb4!!=SDUdDpWl7!r+m6?1D zj+B3J!cJHu+@)wTIHLO_tj0rwi=OZzL2%LIYNR@%-!%WvIO~3fccohNVy&4BRFE`R zb)T0LWft1Ccrp}3(Ortiq)U0zA1C{%jF3Iq&4smPvNOjAZaUOnX!xl=B<&a-?RU6n zFbU(TYN4}xv*^YbepbwclKm=DtxI)%@!?v|R@e2j80EiJX4)$GAaGZ~h+nc*a zsE|u+ldE@tZ*;g0yfNnP-+u=;Q6sZ^M1do86;rlEaRLJpzz~K|e>~UN_0dDaXsXu0Q2{Iq;ml|i`yqvwjBUh>)p%pdAnl)z zRR@H}C4e3JQ6KQS1YvLXX2~t*OXOAKG6oXQ+f-mN^Ob$2VJTpbA6{xiq|VCDWD$QS znRLNb4KE}^PzLTl$K=A%d7iz&5@8l z&)QT`Ld`~wmBTC~NoONq>apU}U~r4?fnwU?o9>EdpG_PUT(Lp{G`|c^OCzk@XgWV) zc4{h`f_H&FM#N86NtmENS|RV&wJ;*NTCz!;M$(K(HC%z|;EJNiro3}hZs$CcQ5D0z zP=RB`gMcp-Ro2t_w+*r28Q9k(#4Uk* zEfaBQ9a-<^g5NIanAaIcjfU91_g{NRcNhZ<u-u*rrg1PtU9Gi1LY2Sm>3%7&5?qb>E zXtEO$2j+CPoOV#4oivIQ?sttG?atwSFww4kS}Ij7i80w%yD$l?CR~wFzGs#h49k!c z!H=VUTskkN<3w2(!kl5E-2DMoKM!}q?|ud z{FaleX@n^x>-APdtLvEg;NB>6dc6{x-L4Sd8Rh+pEkAy;;-jzt%j$c2v9z_Zv>~dO;(d-g-%q;;m?tM-!q9_=eYG79@5wM;{+xb9V#CqX65z zZDLrX+vpas_ zFoXt8D<@0Z+g~!i^tfVrr|?) zhn1_fv1;5*2N~Avh`r(Ab17Xax7?R~8@yWghtNs-YW4k+4&jexI{2|)T4rkYgPc%~ z$C4=c_Ov7Z;MbdN694t-xaaY zn;*I|DH-eXl&S|WDV>dwyv`o6Y2nA&p{_aCs-7Z}qOI*3W+{dNa*lRMyRrG=RqO<3 zG>{ml*GrM*#x>D|RlHyLOkqTNWr6t zxt_8N%_jx~8EDEq`frDdv*2Iw+xrb(#|?8?XvVi{4lCe9#6ng^ z;+={SejQhxeAXVpnX6B9aT1|&U-_|DsTch}J0(a-27)F)Qks5tkN0pZg{D>UpjJqa zR=^SS`2HYO2;|zMnFa_-Y5Z>ar{QlV7`{% zV5S-eh$2NLa+hqFzWKMGUu-vb+T48mBe=U4@Je2SEK4agPh^!fg>{OOy$yTzYEMu}8N|+c#cE&Ct3S~rv5i^)chRQM{ zOL$ANO^C6S5C)BX`8_`0-#_!u^LowmoO9pzxzD-IbsedY{aBzFvLH+dh>F5LiBY8x zAJMi;320P?Oye}~z1vhxsOLbwIte;!b4M8w_GwEm#@uYymO}+-_c>PszNbmA|BxXn ze}?WTvc5#knFQ`L(iCi`7lx_w^@bfpS=7sa!2*iTObMpIBalY#(Dp8`HeKy;89@#z z)CCvzo=%}`JwWu9sXa0`gWup#0rKcF0J*XE7Xc1aasFFumJ)o<-733x!z-Um>K$;0TEWJPOv*l zvN-%V#o~D7Iqpy&G4A6UD-1{10{mG!Zf%CtR3Hn{kilre65=^j@Vx{hPx~qROB}kX z5%l-^LJ>Jgo}W$&y14PE(_1%d({8WTi9uy) z>`3Q=_{vD6@qsRlNrO{Y^r4dhCz|AJ)>P3OtHtJ$XW)pd{>O1TqlL5GCKH|JSko~g z_e#D~d)Zr9Uo8^Ul!a9O?audCpe&Xsp__}up0Ble`1kdswaV+Tt27{d=vKWIgaIX! zRKE)V8j5T^W%<++!jE2gA z+^=9Qe@!2W;-{9NoD1MW_-lN8Jg%y;{YlPKca~ap>3x!=)fIsdlOO+ ztcw?Az-o%h7#;Lk-bHIc*Qx&hZJZ%HJy&WAs6p7tR$^R{#rsWbV9c_8QP2`St=)HdUFXqhpN_@jc zG;Tz}-pr&LLSL$A1w1p2Dbib68k!Czr14%Z2LdWm6)87^dAh&xlBhh2JMuZ}WZRs^ zYE@4_}QZ+O5g5}%${?v>m^~g7SRzY1g^Z6E27a4N(L z?ufvZPaUNV3x`d>ZGNhglrVFzPU{Ls-wc=jP;;b4-YcbZ*5+0d(n5eQSrh_;Q%0?2 z_;X{dhFydUyj$?lElFB{=jDwtQnQdt5A8fI_aK0D3kI*^cYdmeYGB{7zNYfZ5C(=9 z@EXo9;(k^U_RA?0Vwxs%7SS+&CNic*5YaGq4R1F3dhcF_MJ=Fq0kh2A5+3a>O6VbRT!{u+ zdn5X-?xEB|viZUyndNnQx<%`Iw(-tChbjzlOx9e4mAjOSM`7NrIU-Dr4B{+nyQyby z>^`yfG2qs>S%pTP1}Lw9IXrzRGy#IuAqiU7{aS1oXViiBoOe6;1?5V)VkS2#&nO~L z%Xpj%^WK2(2JSHHiU)j&V<3jiwCgwDP{gZ`$}XEp=7ICL0P(7YF+@zPh7&uv+*-wf z_|=X^xh3!rz8`e=tX%voi2JeMP2y#V@c7jczXT6>g*D6odLY~C)bMt#ur|%1^<=ogG=E+opX92aJ0Wjb-pux2W7o@!V*~pOQX?roJGwN=C|6sra zRw#{0!>Cj0t1M0Fqyc zXA9mPUON9L$jOs$&%JynFBzO?dSU_@Fu^ZAY1%RezAvfWt9_3N&&wqoS z@<;;>(PcC}+7r&Zdif#4QDzUJ@pW7@h8G4hyiZBh zAY%Hi;UfQJd3t*CASgP4e|uZ>uK!qw?X-s$61W?v;-G^RB%j_RV{rOJ1G%C5G%3mF z=nC!(UJ{vaq6~gf0{lIv`rGz5xc9KB_4Cv~pfCi{Sk7a6mr#!I{L0;EtAqg{dpQ4l z|FzCHolb2!Ng~!_6E2IeEJ)}2%~T3xcG3UMeS125{p0R@pmU56uoDDptM=9Lm^o+K z+=3wK`B}pje7;oMSL&Jrb6YgdLV%`PXCy?@@E$M0Ua&QEf#!cy`4gMX~?QBeA zKD5}FER%EK#gkbBa?UoS%Oq`BJQhF7#B1Zd|BJ*XLv$(J*&;77{^KJ2ECg=VyU6r_ zib2Y-*t)H%hH6K+8@_i;mve$-pvIvBs4oaIZNA4@MIX~N`F3Q9&LI?czkbx=peab; zdJb-7BK_MF^VevjGUJW=*9ESn4!H$puG zYU?I2Ui-z^jRrNSU`Ct zFRQW<+ZS3nU#2Gm9K!VU%LQ=d!%4xA{cIA^@8GbLR4GGTeyoMgLC=(zDA|{Gr8FM6 z%FYPF?yoDLw7?w=-%VDc>iXF*z-G`HZPW-%ee!W&CxLoYjn9}|Ob7!~@0+D` z;WI%>qNYO1f#KT|-SGbmv_nFS>5gw&jmfaXep&-bM22|(#c!SwS86{01fA5oH&=dn z2I!~pMP5+GieP#+i6xWl+6t4y1?%@XUk}`63QC~_$|9;C7IQQ10fjCY>UBnE0Hl+u zALPdTHx+Q~U~+>ByZtTqY=d+B0vzTia@p+|tEMiDD$3^j&Zj;?oJ-X+kc&WFWxs5K zn+c+l85%C1yH~X@e=HTJRp zTGWvbtY?3SiDnoq@`RRQj+|bgt(Ochp=Ymw0+3&p%YTfoOrhET146`2-1RVt2dv@j zA69hRArUcQ`Wo^b;rXO8K-{Aj#MQ%du?|7FaclJ)2p# z2DTZD*;fYnKi~R#eGqpV8>&dvhOTQ#4boMWTrEI5Zr1bk3`St(vva3kB~LcVsZ2ssoGOY*7Lj}b+nrIMC6&L zeM{U*q=XBgclW|5G(U1|z5aWO>2wO^E8P&b?0tx`qm#q{7F{sSfade%{=?75a!_CD zlP9I*%5&KrN7hv9c1NnzW37ass+_ORS^~8uj=*wp*qp4Hj#jEQvk&~n7jy$K>cr%w ze)ilEDOZMw?aov0t-V``Lbin5=vuk6xUJw?|C{3+N?61`DfEhx0cuV^zD|MZQ!1ra zZ0BuyHp&;jR1J|K9$qN;Pi5qqZo9@thu08;b^-?chPq|^-bJC?OXmi~r2_<@RsVMn+lsV(6& zY~=lcD@T#oIq;>EwV6NWlm|-PH|}KiI?$iI?lz+|nj`Ed%4lZHq1_&34`2Xx$TIm4 znkZxs=_SGOn=3joRC5LZMZ2P!En;H0Cq+Ug`V+k?PJj(KA~;b)3bRvPmj_UjqMt7Y z9pe4cmjA@y$M^qRm61}slrb9=>c@s#I|cmU--waX_14=#7B1`Q<2IIiIAkD-#zfvIyx6X%fi!m>-jVDh}RKYdCDXz}Z}5?Y zrc$R^Ok?nam}SN`Ucyf>^-#qlKmzF{HX+S5`3SA7p2e^Vib|V|? zXC{US)s=5^sy=ZT*@O!L9Q_|pNs=;+s%&cQKjlVWlh?vh(xA*G*&fvVw|8KE416mxsCedzuleEX{{x6GN?ZB=|e!p_Q zkKnFQR}owv&N+JAU9p&ajmehZFp`CqL%nOWF$c`%^kST3}dCL z*46cfv$}B#6~XQYMk+E>?XS0k)u1+~iMipD&^wpS(?|mnNj4uI_9u}aQUsc|Z&qGdbO&Fr^R#1v%D=NehUhq+y?`bFVxjp`ORuC|j9Z&}A~KfuVb{30FquLGMmNQKGS2~WI)fRElWM1+uye4=Q5UF? zq3`?7%V~fE_7f}>9gPERh_%~_Go$TlecF$mvmqI+A@{7?+M~7_+68Ynf|m8p6MLzX z)0ce^rK2+PV6T%1*;y`ueNTbsi!V$NQX5>Z>bH_%<|9!+KuD!w*;CgAXcHIXeZ_G< z)-MTk2Jq9OkWXJ^7oz$It>SqePJ*qEDo)Wr1rn;t-*y5!s_!b92F70ir0Vjv-u7R< z6Vx^2Ei}x$Dyhve)vf#R`e#7qeZ68Ya>bU8abT%j`!@spXt65byiuN)=@yy*y5;AN=A=3R zEcH7dO?!fjp5BFNQ9{Wn!F^BKbTa>8v)k`=V8D~P_8T6 zvaWZnNDONbzm>4AGWnYZdOV`9Mk?%Z{|@onQ=+OMIYp*1D;)nZxswZq;<$FS z+2;HNChti+c8seilW)4cD`!h68nDF*mK2lM___pdPqXR7TN)s8dST{H%Q(dr-Is6zfw$}=ElmL}!>SpU#L3z<9+2J>DS zOGF>1P=soPv54t2=(`_p)9PivDP(SkXpIKluxHD3IiKN}2r;>EX#GJ$s2XL><?pjtm= z8%7^D6$+%cea8E5)by;*n7F7gYJn4*Mi{!`d~;@I-!W1!upPG;p$jvShgM9uC3qAYYLLcRu2CIhxpXLvH$3DsC$o8Ff1$_9d zxpzt#_Vn09E3B0DZz8?=pJtVNrfTteIj^KZ&oiI^$t(S9>}DVtsGF7dTez0HDx)+C zI{C}~904-O^|IOdM*ysTH$FnJevM$+FwHQscTsP_1vhxi)bv84y7P2s6N42r9k<$i zV2g-ax@X&^&r9LJO4!?JKU5<_gH?uw=6=E8u|Wq>cn7Wn5cce;{}J}Uc8K5bSfj1> zy_}{v@W<}nOvEb~js8?S`f z{fR)3`9xGZmAb;O4iPtgdEy$qWm=NyW}iTfr&c&{1LK}lBRvN*IhT`QdsOSZB$lXz z3mHotP1pK2bvpXd?rZNkD`;jGL);prspNAtA&xC(@!L@hw`f95v93p{VbElzmPlOs z@%=7$iw2T8r8jLYcenb5U9v|Q=@1sw0ZdK0biS5VCD(|3=hSGq%Z9J*Ut`7b)T47cU)({N>Ip!NF8A7_ndUz)iUZEX#hk(tV)dCR7_Qwn;2oE0s>3rC#-8Cn|lP{&IY=NSK7C z{HqTn^Nb6{eusO7(9TRM?dZq6>_+1U62)8GVJW0-&bIk4CMvljk}i>SMO@G=LshO; zoj8GF5!~wS3`oBM?LOtQWb`*BW#d=A9wBHtxboREAR`Q_wk)vsop~`U3??m zixVL=c?+E^ob@ Date: Wed, 29 Jan 2020 21:05:34 +0800 Subject: [PATCH 215/257] Makefile: optiomize jsonnet-format target (#2076) Signed-off-by: Xiang Dai <764524258@qq.com> --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 07acaaf1f5..0ac5760973 100644 --- a/Makefile +++ b/Makefile @@ -362,6 +362,9 @@ JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-st .PHONY: jsonnet-format jsonnet-format: + @which jsonnetfmt 2>/dev/null || ( \ + echo "Cannot find jsonnetfmt command, please install from https://github.com/google/jsonnet/releases. If your C++ does not support GLIBCXX_3.4.20, please use xxx-in-container target like jsonnet-format-in-container." && exit 1 + ) find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \ xargs -n 1 -- $(JSONNET_FMT) -i From 4513d0d311589a0ccbb8d395b0e69be7eec29dca Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Thu, 30 Jan 2020 10:47:30 +0100 Subject: [PATCH 216/257] mixin: Fix querier store DNS query alert (#2034) * Fix querier store DNS alert percentage issue Signed-off-by: Kemal Akkoyun * Generate files Signed-off-by: Kemal Akkoyun --- examples/alerts/alerts.md | 25 +++++++++++------------ examples/alerts/alerts.yaml | 29 +++++++++++++-------------- mixin/thanos/alerts/querier.libsonnet | 24 ++++++++++------------ 3 files changed, 37 insertions(+), 41 deletions(-) diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index 3548287a63..b45f78d8c1 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -279,7 +279,7 @@ name: thanos-querier.rules rules: - alert: ThanosQuerierHttpRequestQueryErrorRateHigh annotations: - message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests. expr: | ( @@ -292,7 +292,7 @@ rules: severity: critical - alert: ThanosQuerierHttpRequestQueryRangeErrorRateHigh annotations: - message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests. expr: | ( @@ -305,7 +305,7 @@ rules: severity: critical - alert: ThanosQuerierGrpcServerErrorRate annotations: - message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests. expr: | ( @@ -319,34 +319,33 @@ rules: severity: warning - alert: ThanosQuerierGrpcClientErrorRate annotations: - message: Thanos Query {{$labels.job}} is failing to send {{ $value | humanize + message: Thanos Querier {{$labels.job}} is failing to send {{ $value | humanize }}% of requests. expr: | ( sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job=~"thanos-querier.*"}[5m])) / sum by (job) (rate(grpc_client_started_total{job=~"thanos-querier.*"}[5m])) - * 100 > 5 - ) + ) * 100 > 5 for: 5m labels: severity: warning - alert: ThanosQuerierHighDNSFailures annotations: - message: Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries. + message: Thanos Queriers {{$labels.job}} have {{ $value | humanize }}% of failing + DNS queries for store endpoints. expr: | ( - sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) / - sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) - > 1 - ) + sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + ) * 100 > 1 for: 15m labels: severity: warning - alert: ThanosQuerierInstantLatencyHigh annotations: - message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value + message: Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries. expr: | ( @@ -359,7 +358,7 @@ rules: severity: critical - alert: ThanosQuerierRangeLatencyHigh annotations: - message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value + message: Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries. expr: | ( diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index 73579f897a..a1e77d932a 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -55,7 +55,7 @@ groups: rules: - alert: ThanosQuerierHttpRequestQueryErrorRateHigh annotations: - message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests. expr: | ( @@ -68,7 +68,7 @@ groups: severity: critical - alert: ThanosQuerierHttpRequestQueryRangeErrorRateHigh annotations: - message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests. expr: | ( @@ -81,7 +81,7 @@ groups: severity: critical - alert: ThanosQuerierGrpcServerErrorRate annotations: - message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests. expr: | ( @@ -95,35 +95,34 @@ groups: severity: warning - alert: ThanosQuerierGrpcClientErrorRate annotations: - message: Thanos Query {{$labels.job}} is failing to send {{ $value | humanize + message: Thanos Querier {{$labels.job}} is failing to send {{ $value | humanize }}% of requests. expr: | ( sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job=~"thanos-querier.*"}[5m])) / sum by (job) (rate(grpc_client_started_total{job=~"thanos-querier.*"}[5m])) - * 100 > 5 - ) + ) * 100 > 5 for: 5m labels: severity: warning - alert: ThanosQuerierHighDNSFailures annotations: - message: Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries. + message: Thanos Queriers {{$labels.job}} have {{ $value | humanize }}% of failing + DNS queries for store endpoints. expr: | ( - sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) / - sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) - > 1 - ) + sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + ) * 100 > 1 for: 15m labels: severity: warning - alert: ThanosQuerierInstantLatencyHigh annotations: - message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value - }} seconds for instant queries. + message: Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ + $value }} seconds for instant queries. expr: | ( histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"})) > 10 @@ -135,8 +134,8 @@ groups: severity: critical - alert: ThanosQuerierRangeLatencyHigh annotations: - message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value - }} seconds for instant queries. + message: Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ + $value }} seconds for instant queries. expr: | ( histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query_range"})) > 10 diff --git a/mixin/thanos/alerts/querier.libsonnet b/mixin/thanos/alerts/querier.libsonnet index 38705aacf3..ca3ae2a218 100644 --- a/mixin/thanos/alerts/querier.libsonnet +++ b/mixin/thanos/alerts/querier.libsonnet @@ -12,7 +12,7 @@ { alert: 'ThanosQuerierHttpRequestQueryErrorRateHigh', annotations: { - message: 'Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests.', + message: 'Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests.', }, expr: ||| ( @@ -29,7 +29,7 @@ { alert: 'ThanosQuerierHttpRequestQueryRangeErrorRateHigh', annotations: { - message: 'Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests.', + message: 'Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests.', }, expr: ||| ( @@ -46,7 +46,7 @@ { alert: 'ThanosQuerierGrpcServerErrorRate', annotations: { - message: 'Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', + message: 'Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', }, expr: ||| ( @@ -64,15 +64,14 @@ { alert: 'ThanosQuerierGrpcClientErrorRate', annotations: { - message: 'Thanos Query {{$labels.job}} is failing to send {{ $value | humanize }}% of requests.', + message: 'Thanos Querier {{$labels.job}} is failing to send {{ $value | humanize }}% of requests.', }, expr: ||| ( sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", %(selector)s}[5m])) / sum by (job) (rate(grpc_client_started_total{%(selector)s}[5m])) - * 100 > 5 - ) + ) * 100 > 5 ||| % thanos.querier, 'for': '5m', labels: { @@ -82,15 +81,14 @@ { alert: 'ThanosQuerierHighDNSFailures', annotations: { - message: 'Thanos Querys {{$labels.job}} have {{ $value }} of failing DNS queries.', + message: 'Thanos Queriers {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for store endpoints.', }, expr: ||| ( - sum by (job) (rate(thanos_query_store_apis_dns_failures_total{%(selector)s}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{%(selector)s}[5m])) / - sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{%(selector)s}[5m])) - > 1 - ) + sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{%(selector)s}[5m])) + ) * 100 > 1 ||| % thanos.querier, 'for': '15m', labels: { @@ -100,7 +98,7 @@ { alert: 'ThanosQuerierInstantLatencyHigh', annotations: { - message: 'Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', + message: 'Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', }, expr: ||| ( @@ -117,7 +115,7 @@ { alert: 'ThanosQuerierRangeLatencyHigh', annotations: { - message: 'Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', + message: 'Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', }, expr: ||| ( From 499eef0e1fa58945eb2978c1cfd989ed3ba114ea Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Thu, 30 Jan 2020 15:27:48 +0000 Subject: [PATCH 217/257] testutil: Moving all e2e utils to separate package, added TB union. (#2077) Signed-off-by: Bartlomiej Plotka --- cmd/thanos/main_test.go | 11 ++- pkg/block/block_test.go | 17 +++-- pkg/block/index_test.go | 3 +- pkg/block/indexheader/header_test.go | 7 +- pkg/compact/compact_e2e_test.go | 3 +- pkg/promclient/promclient_e2e_test.go | 11 +-- pkg/query/api/v1_test.go | 3 +- pkg/shipper/shipper_e2e_test.go | 3 +- pkg/store/bucket.go | 2 +- pkg/store/bucket_e2e_test.go | 5 +- pkg/store/bucket_test.go | 9 ++- pkg/store/prometheus_test.go | 13 ++-- pkg/store/tsdb_test.go | 11 +-- pkg/testutil/{ => e2eutil}/copy.go | 5 +- pkg/testutil/{ => e2eutil}/port.go | 2 +- pkg/testutil/{ => e2eutil}/prometheus.go | 7 +- pkg/testutil/{ => e2eutil}/sysprocattr.go | 2 +- .../{ => e2eutil}/sysprocattr_linux.go | 2 +- pkg/testutil/testorbench.go | 75 +++++++++++++++++++ pkg/testutil/testorbench_test.go | 46 ++++++++++++ test/e2e/spinup_test.go | 10 +-- test/e2e/store_gateway_test.go | 7 +- 22 files changed, 194 insertions(+), 60 deletions(-) rename pkg/testutil/{ => e2eutil}/copy.go (89%) rename pkg/testutil/{ => e2eutil}/port.go (95%) rename pkg/testutil/{ => e2eutil}/prometheus.go (98%) rename pkg/testutil/{ => e2eutil}/sysprocattr.go (91%) rename pkg/testutil/{ => e2eutil}/sysprocattr_linux.go (94%) create mode 100644 pkg/testutil/testorbench.go create mode 100644 pkg/testutil/testorbench_test.go diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index 839c125e07..1917a79f20 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -9,9 +9,6 @@ import ( "os" "path" "path/filepath" - - "github.com/thanos-io/thanos/pkg/block/metadata" - "testing" "time" @@ -21,10 +18,12 @@ import ( promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" ) func TestCleanupIndexCacheFolder(t *testing.T) { @@ -41,7 +40,7 @@ func TestCleanupIndexCacheFolder(t *testing.T) { // Upload one compaction lvl = 2 block, one compaction lvl = 1. // We generate index cache files only for lvl > 1 blocks. { - id, err := testutil.CreateBlock( + id, err := e2eutil.CreateBlock( ctx, dir, []labels.Labels{{{Name: "a", Value: "1"}}}, @@ -59,7 +58,7 @@ func TestCleanupIndexCacheFolder(t *testing.T) { testutil.Ok(t, block.Upload(ctx, logger, bkt, path.Join(dir, id.String()))) } { - id, err := testutil.CreateBlock( + id, err := e2eutil.CreateBlock( ctx, dir, []labels.Labels{{{Name: "a", Value: "1"}}}, @@ -102,7 +101,7 @@ func TestCleanupDownsampleCacheFolder(t *testing.T) { bkt := inmem.NewBucket() var id ulid.ULID { - id, err = testutil.CreateBlock( + id, err = e2eutil.CreateBlock( ctx, dir, []labels.Labels{{{Name: "a", Value: "1"}}}, diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index 38b649afe2..e6bdfc8ce1 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" "github.com/oklog/ulid" ) @@ -77,7 +78,7 @@ func TestUpload(t *testing.T) { defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() bkt := inmem.NewBucket() - b1, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + b1, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, @@ -105,7 +106,7 @@ func TestUpload(t *testing.T) { testutil.NotOk(t, err) testutil.Assert(t, strings.HasSuffix(err.Error(), "/meta.json: no such file or directory"), "") } - testutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename)) + e2eutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename)) { // Missing chunks. err := Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String())) @@ -116,7 +117,7 @@ func TestUpload(t *testing.T) { testutil.Equals(t, 1, len(bkt.Objects())) } testutil.Ok(t, os.MkdirAll(path.Join(tmpDir, "test", b1.String(), ChunksDirname), os.ModePerm)) - testutil.Copy(t, path.Join(tmpDir, b1.String(), ChunksDirname, "000001"), path.Join(tmpDir, "test", b1.String(), ChunksDirname, "000001")) + e2eutil.Copy(t, path.Join(tmpDir, b1.String(), ChunksDirname, "000001"), path.Join(tmpDir, "test", b1.String(), ChunksDirname, "000001")) { // Missing index file. err := Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String())) @@ -126,7 +127,7 @@ func TestUpload(t *testing.T) { // Only debug meta.json present. testutil.Equals(t, 1, len(bkt.Objects())) } - testutil.Copy(t, path.Join(tmpDir, b1.String(), IndexFilename), path.Join(tmpDir, "test", b1.String(), IndexFilename)) + e2eutil.Copy(t, path.Join(tmpDir, b1.String(), IndexFilename), path.Join(tmpDir, "test", b1.String(), IndexFilename)) testutil.Ok(t, os.Remove(path.Join(tmpDir, "test", b1.String(), MetaFilename))) { // Missing meta.json file. @@ -137,7 +138,7 @@ func TestUpload(t *testing.T) { // Only debug meta.json present. testutil.Equals(t, 1, len(bkt.Objects())) } - testutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename)) + e2eutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename)) { // Full block. testutil.Ok(t, Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()))) @@ -156,7 +157,7 @@ func TestUpload(t *testing.T) { } { // Upload with no external labels should be blocked. - b2, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + b2, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, @@ -182,7 +183,7 @@ func TestDelete(t *testing.T) { bkt := inmem.NewBucket() { - b1, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + b1, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, @@ -199,7 +200,7 @@ func TestDelete(t *testing.T) { testutil.Equals(t, 1, len(bkt.Objects())) } { - b2, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + b2, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, diff --git a/pkg/block/index_test.go b/pkg/block/index_test.go index 57860461b7..7be4e75ea1 100644 --- a/pkg/block/index_test.go +++ b/pkg/block/index_test.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" ) func TestRewrite(t *testing.T) { @@ -26,7 +27,7 @@ func TestRewrite(t *testing.T) { testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() - b, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + b, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, diff --git a/pkg/block/indexheader/header_test.go b/pkg/block/indexheader/header_test.go index 932071482c..c0cc76d3f8 100644 --- a/pkg/block/indexheader/header_test.go +++ b/pkg/block/indexheader/header_test.go @@ -25,6 +25,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/filesystem" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" ) func TestReaders(t *testing.T) { @@ -39,7 +40,7 @@ func TestReaders(t *testing.T) { defer func() { testutil.Ok(t, bkt.Close()) }() // Create block index version 2. - id1, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + id1, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, @@ -69,7 +70,7 @@ func TestReaders(t *testing.T) { m, err := metadata.Read("./testdata/index_format_v1") testutil.Ok(t, err) - testutil.Copy(t, "./testdata/index_format_v1", filepath.Join(tmpDir, m.ULID.String())) + e2eutil.Copy(t, "./testdata/index_format_v1", filepath.Join(tmpDir, m.ULID.String())) _, err = metadata.InjectThanos(log.NewNopLogger(), filepath.Join(tmpDir, m.ULID.String()), metadata.Thanos{ Labels: labels.Labels{{Name: "ext1", Value: "1"}}.Map(), @@ -264,7 +265,7 @@ func prepareIndexV2Block(t testing.TB, tmpDir string, bkt objstore.Bucket) *meta m, err := metadata.Read("./testdata/index_format_v2") testutil.Ok(t, err) - testutil.Copy(t, "./testdata/index_format_v2", filepath.Join(tmpDir, m.ULID.String())) + e2eutil.Copy(t, "./testdata/index_format_v2", filepath.Join(tmpDir, m.ULID.String())) _, err = metadata.InjectThanos(log.NewNopLogger(), filepath.Join(tmpDir, m.ULID.String()), metadata.Thanos{ Labels: labels.Labels{{Name: "ext1", Value: "1"}}.Map(), diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 80dba2a795..56869e5bdc 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -30,6 +30,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/objtesting" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" ) func TestSyncer_GarbageCollect_e2e(t *testing.T) { @@ -383,7 +384,7 @@ func createAndUpload(t testing.TB, bkt objstore.Bucket, blocks []blockgenSpec) ( if b.numSamples == 0 { id, err = createEmptyBlock(prepareDir, b.mint, b.maxt, b.extLset, b.res) } else { - id, err = testutil.CreateBlock(ctx, prepareDir, b.series, b.numSamples, b.mint, b.maxt, b.extLset, b.res) + id, err = e2eutil.CreateBlock(ctx, prepareDir, b.series, b.numSamples, b.mint, b.maxt, b.extLset, b.res) } testutil.Ok(t, err) diff --git a/pkg/promclient/promclient_e2e_test.go b/pkg/promclient/promclient_e2e_test.go index 61f5a5e85a..f692ef11fe 100644 --- a/pkg/promclient/promclient_e2e_test.go +++ b/pkg/promclient/promclient_e2e_test.go @@ -20,10 +20,11 @@ import ( "github.com/prometheus/prometheus/pkg/timestamp" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" ) func TestIsWALFileAccessible_e2e(t *testing.T) { - testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) { + e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) { testutil.Ok(t, p.Start()) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) @@ -36,7 +37,7 @@ func TestIsWALFileAccessible_e2e(t *testing.T) { } func TestExternalLabels_e2e(t *testing.T) { - testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) { + e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) { testutil.Ok(t, p.SetConfig(` global: external_labels: @@ -59,7 +60,7 @@ global: } func TestConfiguredFlags_e2e(t *testing.T) { - testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) { + e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) { testutil.Ok(t, p.Start()) u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) @@ -78,12 +79,12 @@ func TestConfiguredFlags_e2e(t *testing.T) { } func TestSnapshot_e2e(t *testing.T) { - testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) { + e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) { now := time.Now() ctx := context.Background() // Create artificial block. - id, err := testutil.CreateBlockWithTombstone( + id, err := e2eutil.CreateBlockWithTombstone( ctx, p.Dir(), []labels.Labels{labels.FromStrings("a", "b")}, diff --git a/pkg/query/api/v1_test.go b/pkg/query/api/v1_test.go index 80b602bec7..481754f3ac 100644 --- a/pkg/query/api/v1_test.go +++ b/pkg/query/api/v1_test.go @@ -45,6 +45,7 @@ import ( "github.com/thanos-io/thanos/pkg/query" "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" ) func TestEndpoints(t *testing.T) { @@ -85,7 +86,7 @@ func TestEndpoints(t *testing.T) { }, } - db, err := testutil.NewTSDB() + db, err := e2eutil.NewTSDB() defer func() { testutil.Ok(t, db.Close()) }() testutil.Ok(t, err) diff --git a/pkg/shipper/shipper_e2e_test.go b/pkg/shipper/shipper_e2e_test.go index 2b17e460e7..19695426f2 100644 --- a/pkg/shipper/shipper_e2e_test.go +++ b/pkg/shipper/shipper_e2e_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/thanos-io/thanos/pkg/objstore/inmem" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" "github.com/go-kit/kit/log" "github.com/oklog/ulid" @@ -172,7 +173,7 @@ func TestShipper_SyncBlocks_e2e(t *testing.T) { } func TestShipper_SyncBlocksWithMigrating_e2e(t *testing.T) { - testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) { + e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) { dir, err := ioutil.TempDir("", "shipper-e2e-test") testutil.Ok(t, err) defer func() { diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index bdeaaf2f9e..8b09feafc9 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -1703,7 +1703,7 @@ type partitioner interface { // Partition partitions length entries into n <= length ranges that cover all // input ranges // It supports overlapping ranges. - // NOTE: It expects range to be sorted by start time. + // NOTE: It expects range to be ted by start time. Partition(length int, rng func(int) (uint64, uint64)) []part } diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index 77bd8dd21a..6bdc6a5950 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -26,6 +26,7 @@ import ( storecache "github.com/thanos-io/thanos/pkg/store/cache" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" ) var ( @@ -100,9 +101,9 @@ func prepareTestBlocks(t testing.TB, now time.Time, count int, dir string, bkt o // Create two blocks per time slot. Only add 10 samples each so only one chunk // gets created each. This way we can easily verify we got 10 chunks per series below. - id1, err := testutil.CreateBlock(ctx, dir, series[:4], 10, mint, maxt, extLset, 0) + id1, err := e2eutil.CreateBlock(ctx, dir, series[:4], 10, mint, maxt, extLset, 0) testutil.Ok(t, err) - id2, err := testutil.CreateBlock(ctx, dir, series[4:], 10, mint, maxt, extLset, 0) + id2, err := e2eutil.CreateBlock(ctx, dir, series[4:], 10, mint, maxt, extLset, 0) testutil.Ok(t, err) dir1, dir2 := filepath.Join(dir, id1.String()), filepath.Join(dir, id2.String()) diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 216adedc09..a277f392cf 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -35,6 +35,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" "gopkg.in/yaml.v2" ) @@ -502,19 +503,19 @@ func TestBucketStore_Sharding(t *testing.T) { bkt := inmem.NewBucket() series := []labels.Labels{labels.FromStrings("a", "1", "b", "1")} - id1, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0) + id1, err := e2eutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0) testutil.Ok(t, err) testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id1.String()))) - id2, err := testutil.CreateBlock(ctx, dir, series, 10, 1000, 2000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0) + id2, err := e2eutil.CreateBlock(ctx, dir, series, 10, 1000, 2000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0) testutil.Ok(t, err) testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id2.String()))) - id3, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "b"}, {Name: "region", Value: "r1"}}, 0) + id3, err := e2eutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "b"}, {Name: "region", Value: "r1"}}, 0) testutil.Ok(t, err) testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id3.String()))) - id4, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r2"}}, 0) + id4, err := e2eutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r2"}}, 0) testutil.Ok(t, err) testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id4.String()))) diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index d2bb97b06c..703d52f687 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/store/storepb" @@ -37,7 +38,7 @@ func testPrometheusStoreSeriesE2e(t *testing.T, prefix string) { defer leaktest.CheckTimeout(t, 10*time.Second)() - p, err := testutil.NewPrometheusOnPath(prefix) + p, err := e2eutil.NewPrometheusOnPath(prefix) testutil.Ok(t, err) defer func() { testutil.Ok(t, p.Stop()) }() @@ -189,7 +190,7 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() - p, err := testutil.NewPrometheus() + p, err := e2eutil.NewPrometheus() testutil.Ok(t, err) defer func() { testutil.Ok(t, p.Stop()) }() @@ -291,7 +292,7 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { func TestPrometheusStore_LabelValues_e2e(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() - p, err := testutil.NewPrometheus() + p, err := e2eutil.NewPrometheus() testutil.Ok(t, err) defer func() { testutil.Ok(t, p.Stop()) }() @@ -327,7 +328,7 @@ func TestPrometheusStore_LabelValues_e2e(t *testing.T) { func TestPrometheusStore_ExternalLabelValues_e2e(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() - p, err := testutil.NewPrometheus() + p, err := e2eutil.NewPrometheus() testutil.Ok(t, err) defer func() { testutil.Ok(t, p.Stop()) }() @@ -367,7 +368,7 @@ func TestPrometheusStore_ExternalLabelValues_e2e(t *testing.T) { func TestPrometheusStore_Series_MatchExternalLabel_e2e(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() - p, err := testutil.NewPrometheus() + p, err := e2eutil.NewPrometheus() testutil.Ok(t, err) defer func() { testutil.Ok(t, p.Stop()) }() @@ -503,7 +504,7 @@ func testSeries_SplitSamplesIntoChunksWithMaxSizeOfUint16_e2e(t *testing.T, appe func TestPrometheusStore_Series_SplitSamplesIntoChunksWithMaxSizeOfUint16_e2e(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() - p, err := testutil.NewPrometheus() + p, err := e2eutil.NewPrometheus() testutil.Ok(t, err) defer func() { testutil.Ok(t, p.Stop()) }() diff --git a/pkg/store/tsdb_test.go b/pkg/store/tsdb_test.go index d8ef8eda0c..1c256f6b22 100644 --- a/pkg/store/tsdb_test.go +++ b/pkg/store/tsdb_test.go @@ -14,6 +14,7 @@ import ( "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" ) func TestTSDBStore_Info(t *testing.T) { @@ -22,7 +23,7 @@ func TestTSDBStore_Info(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - db, err := testutil.NewTSDB() + db, err := e2eutil.NewTSDB() defer func() { testutil.Ok(t, db.Close()) }() testutil.Ok(t, err) @@ -43,7 +44,7 @@ func TestTSDBStore_Series(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - db, err := testutil.NewTSDB() + db, err := e2eutil.NewTSDB() defer func() { testutil.Ok(t, db.Close()) }() testutil.Ok(t, err) @@ -169,7 +170,7 @@ func TestTSDBStore_LabelNames(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - db, err := testutil.NewTSDB() + db, err := e2eutil.NewTSDB() defer func() { testutil.Ok(t, db.Close()) }() testutil.Ok(t, err) @@ -224,7 +225,7 @@ func TestTSDBStore_LabelValues(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - db, err := testutil.NewTSDB() + db, err := e2eutil.NewTSDB() defer func() { testutil.Ok(t, db.Close()) }() testutil.Ok(t, err) @@ -279,7 +280,7 @@ func TestTSDBStore_LabelValues(t *testing.T) { func TestTSDBStore_Series_SplitSamplesIntoChunksWithMaxSizeOfUint16_e2e(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() - db, err := testutil.NewTSDB() + db, err := e2eutil.NewTSDB() defer func() { testutil.Ok(t, db.Close()) }() testutil.Ok(t, err) diff --git a/pkg/testutil/copy.go b/pkg/testutil/e2eutil/copy.go similarity index 89% rename from pkg/testutil/copy.go rename to pkg/testutil/e2eutil/copy.go index f90dc68bbf..d31c6e9fd5 100644 --- a/pkg/testutil/copy.go +++ b/pkg/testutil/e2eutil/copy.go @@ -1,7 +1,7 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. -package testutil +package e2eutil import ( "io" @@ -10,10 +10,11 @@ import ( "testing" "github.com/pkg/errors" + "github.com/thanos-io/thanos/pkg/testutil" ) func Copy(t testing.TB, src, dst string) { - Ok(t, copyRecursive(src, dst)) + testutil.Ok(t, copyRecursive(src, dst)) } func copyRecursive(src, dst string) error { diff --git a/pkg/testutil/port.go b/pkg/testutil/e2eutil/port.go similarity index 95% rename from pkg/testutil/port.go rename to pkg/testutil/e2eutil/port.go index 482ced0858..986f1c7d7f 100644 --- a/pkg/testutil/port.go +++ b/pkg/testutil/e2eutil/port.go @@ -1,7 +1,7 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. -package testutil +package e2eutil import "net" diff --git a/pkg/testutil/prometheus.go b/pkg/testutil/e2eutil/prometheus.go similarity index 98% rename from pkg/testutil/prometheus.go rename to pkg/testutil/e2eutil/prometheus.go index 10f34b8a56..56972041ce 100644 --- a/pkg/testutil/prometheus.go +++ b/pkg/testutil/e2eutil/prometheus.go @@ -1,7 +1,7 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. -package testutil +package e2eutil import ( "context" @@ -26,6 +26,7 @@ import ( "github.com/prometheus/prometheus/tsdb" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/testutil" "golang.org/x/sync/errgroup" ) @@ -98,10 +99,10 @@ func ForeachPrometheus(t *testing.T, testFn func(t testing.TB, p *Prometheus)) { for _, ver := range strings.Split(vers, " ") { if ok := t.Run(ver, func(t *testing.T) { p, err := newPrometheus(ver, "") - Ok(t, err) + testutil.Ok(t, err) testFn(t, p) - Ok(t, p.Stop()) + testutil.Ok(t, p.Stop()) }); !ok { return } diff --git a/pkg/testutil/sysprocattr.go b/pkg/testutil/e2eutil/sysprocattr.go similarity index 91% rename from pkg/testutil/sysprocattr.go rename to pkg/testutil/e2eutil/sysprocattr.go index 98d8755ce4..cc982a5bd3 100644 --- a/pkg/testutil/sysprocattr.go +++ b/pkg/testutil/e2eutil/sysprocattr.go @@ -3,7 +3,7 @@ // +build !linux -package testutil +package e2eutil import "syscall" diff --git a/pkg/testutil/sysprocattr_linux.go b/pkg/testutil/e2eutil/sysprocattr_linux.go similarity index 94% rename from pkg/testutil/sysprocattr_linux.go rename to pkg/testutil/e2eutil/sysprocattr_linux.go index f7c59a0b50..dd77ed32a1 100644 --- a/pkg/testutil/sysprocattr_linux.go +++ b/pkg/testutil/e2eutil/sysprocattr_linux.go @@ -1,7 +1,7 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. -package testutil +package e2eutil import "syscall" diff --git a/pkg/testutil/testorbench.go b/pkg/testutil/testorbench.go new file mode 100644 index 0000000000..dcfbff11ed --- /dev/null +++ b/pkg/testutil/testorbench.go @@ -0,0 +1,75 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package testutil + +import ( + "testing" +) + +// TB represents union of test and benchmark. +// This allows the same test suite to be run by both benchmark and test, helping to reuse more code. +// The reason is that usually benchmarks are not being run on CI, especially for short tests, so you need to recreate +// usually similar tests for `Test(t *testing.T)` methods. Example of usage is presented here: +// +// func TestTestOrBench(t *testing.T) { +// tb := NewTB(t) +// tb.Run("1", func(tb TB) { testorbenchComplexTest(tb) }) +// tb.Run("2", func(tb TB) { testorbenchComplexTest(tb) }) +// } +// +// func BenchmarkTestOrBench(b *testing.B) { +// tb := NewTB(t) +// tb.Run("1", func(tb TB) { testorbenchComplexTest(tb) }) +// tb.Run("2", func(tb TB) { testorbenchComplexTest(tb) }) +// } +type TB interface { + testing.TB + IsBenchmark() bool + Run(name string, f func(t TB)) bool + N() int + ResetTimer() +} + +// tb implements TB as well as testing.TB interfaces. +type tb struct { + testing.TB +} + +// NewTB creates tb from testing.TB. +func NewTB(t testing.TB) TB { return &tb{TB: t} } + +// Run benchmarks/tests f as a subbenchmark/subtest with the given name. It reports +// whether there were any failures. +// +// A subbenchmark/subtest is like any other benchmark/test. +func (t *tb) Run(name string, f func(t TB)) bool { + if b, ok := t.TB.(*testing.B); ok { + return b.Run(name, func(nested *testing.B) { f(&tb{TB: nested}) }) + } + if t, ok := t.TB.(*testing.T); ok { + return t.Run(name, func(nested *testing.T) { f(&tb{TB: nested}) }) + } + panic("not a benchmark and not a test") +} + +// N returns number of iterations to do for benchmark, 1 in case of test. +func (t *tb) N() int { + if b, ok := t.TB.(*testing.B); ok { + return b.N + } + return 1 +} + +// ResetTimer resets a timer, if it's a benchmark, noop otherwise. +func (t *tb) ResetTimer() { + if b, ok := t.TB.(*testing.B); ok { + b.ResetTimer() + } +} + +// IsBenchmark returns true if it's a benchmark. +func (t *tb) IsBenchmark() bool { + _, ok := t.TB.(*testing.B) + return ok +} diff --git a/pkg/testutil/testorbench_test.go b/pkg/testutil/testorbench_test.go new file mode 100644 index 0000000000..787e98fe66 --- /dev/null +++ b/pkg/testutil/testorbench_test.go @@ -0,0 +1,46 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package testutil + +import "testing" + +func TestTestOrBench(t *testing.T) { + tb := NewTB(t) + tb.Run("1", func(tb TB) { testorbenchComplexTest(tb) }) + tb.Run("2", func(tb TB) { testorbenchComplexTest(tb) }) +} + +func BenchmarkTestOrBench(b *testing.B) { + tb := NewTB(b) + tb.Run("1", func(tb TB) { testorbenchComplexTest(tb) }) + tb.Run("2", func(tb TB) { testorbenchComplexTest(tb) }) +} + +func testorbenchComplexTest(tb TB) { + tb.Run("a", func(tb TB) { + tb.Run("aa", func(tb TB) { + tb.ResetTimer() + for i := 0; i < tb.N(); i++ { + if !tb.IsBenchmark() { + if tb.N() != 1 { + tb.FailNow() + } + } + } + }) + }) + tb.Run("b", func(tb TB) { + tb.Run("bb", func(tb TB) { + tb.ResetTimer() + for i := 0; i < tb.N(); i++ { + if !tb.IsBenchmark() { + if tb.N() != 1 { + tb.FailNow() + } + } + } + }) + }) + +} diff --git a/test/e2e/spinup_test.go b/test/e2e/spinup_test.go index 46e4ae2295..ad0f57b6c4 100644 --- a/test/e2e/spinup_test.go +++ b/test/e2e/spinup_test.go @@ -25,7 +25,7 @@ import ( "github.com/thanos-io/thanos/pkg/receive" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" - "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" "google.golang.org/grpc" ) @@ -94,7 +94,7 @@ func newCmdExec(cmd *exec.Cmd) *cmdExec { func (c *cmdExec) Start(stdout io.Writer, stderr io.Writer) error { c.Stderr = stderr c.Stdout = stdout - c.SysProcAttr = testutil.SysProcAttr() + c.SysProcAttr = e2eutil.SysProcAttr() return c.Cmd.Start() } @@ -138,7 +138,7 @@ func prometheus(http address, config string) *prometheusScheduler { return nil, errors.Wrap(err, "creating prom config failed") } - return newCmdExec(exec.Command(testutil.PrometheusBinary(), + return newCmdExec(exec.Command(e2eutil.PrometheusBinary(), "--config.file", promDir+"/prometheus.yml", "--storage.tsdb.path", promDir, "--storage.tsdb.max-block-duration", "2h", @@ -299,7 +299,7 @@ receivers: if err := ioutil.WriteFile(dir+"/config.yaml", []byte(config), 0666); err != nil { return nil, errors.Wrap(err, "creating alertmanager config file failed") } - return newCmdExec(exec.Command(testutil.AlertmanagerBinary(), + return newCmdExec(exec.Command(e2eutil.AlertmanagerBinary(), "--config.file", dir+"/config.yaml", "--web.listen-address", http.HostPort(), "--cluster.listen-address", "", @@ -418,7 +418,7 @@ func minio(http address, config s3.Config) *serverScheduler { return nil, errors.Wrap(err, "creating minio dir failed") } - cmd := exec.Command(testutil.MinioBinary(), + cmd := exec.Command(e2eutil.MinioBinary(), "server", "--address", http.HostPort(), dbDir, diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index d30bb184ff..484fbec325 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -22,6 +22,7 @@ import ( "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" yaml "gopkg.in/yaml.v2" ) @@ -79,13 +80,13 @@ func TestStoreGateway(t *testing.T) { extLset3 := labels.FromStrings("ext1", "value2", "replica", "3") now := time.Now() - id1, err := testutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset, 0) + id1, err := e2eutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset, 0) testutil.Ok(t, err) - id2, err := testutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset2, 0) + id2, err := e2eutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset2, 0) testutil.Ok(t, err) - id3, err := testutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset3, 0) + id3, err := e2eutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset3, 0) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) From 25f5b0b9290fa1620647f00dd8df954380efc530 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 31 Jan 2020 16:22:13 +0800 Subject: [PATCH 218/257] Makefile: fix quay.io/coreos/jsonnet-ci tag (#2080) Signed-off-by: Xiang Dai <764524258@qq.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0ac5760973..8f7ee0c7a7 100644 --- a/Makefile +++ b/Makefile @@ -316,7 +316,7 @@ JSONNET_CONTAINER_CMD:=docker run --rm \ -w "/go/src/github.com/thanos-io/thanos" \ -e USER=deadbeef \ -e GO111MODULE=on \ - quay.io/coreos/jsonnet-ci + quay.io/coreos/jsonnet-ci:release-0.35 .PHONY: examples-in-container examples-in-container: From 9b17ba18f8bb2d3ad9bd8ce2ac00e18430665655 Mon Sep 17 00:00:00 2001 From: Tom Bartek Date: Fri, 31 Jan 2020 23:51:24 +0100 Subject: [PATCH 219/257] mixin: Fix broken links in docs (#2083) Fix links to examples directory and renamed defaults.libsonnet file Signed-off-by: itsx --- mixin/thanos/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mixin/thanos/README.md b/mixin/thanos/README.md index f5aef3e212..f53e11c509 100644 --- a/mixin/thanos/README.md +++ b/mixin/thanos/README.md @@ -54,7 +54,7 @@ $ jb update #### Configure -This project is intended to be used as a library. You can extend and customize dashboards and alerting rules by creating for own generators, such as the generators ([alerts.jsonnet](alerts.jsonnet) and [dashboards.jsonnet](dashboards.jsonnet)) that are use to create [examples](examples). Default parameters are collected in [defaults.jsonnet](defaults.jsonnet), feel free to modify and generate your own definitons. +This project is intended to be used as a library. You can extend and customize dashboards and alerting rules by creating for own generators, such as the generators ([alerts.jsonnet](alerts.jsonnet) and [dashboards.jsonnet](dashboards.jsonnet)) that are use to create [examples](../../examples). Default parameters are collected in [defaults.libsonnet](defaults.libsonnet), feel free to modify and generate your own definitons. [embedmd]:# (defaults.libsonnet) ```libsonnet @@ -147,7 +147,7 @@ You validate your structural correctness of your Prometheus [alerting rules](htt $ make example-rules-lint ``` -Check out [test.yaml](examples/alerts/tests.yaml) to add/modify tests for the mixin. To learn more about how to write test for Prometheus, check out [official documentation](https://www.prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/). +Check out [test.yaml](../../examples/alerts/tests.yaml) to add/modify tests for the mixin. To learn more about how to write test for Prometheus, check out [official documentation](https://www.prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/). You test alerts with: From 0e7e4bb6f287a6e83cf59369274cd72198a534ac Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 3 Feb 2020 17:01:19 +0000 Subject: [PATCH 220/257] Added naming convention rule for components. (#2094) Signed-off-by: Bartlomiej Plotka --- CONTRIBUTING.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2aca8dd5da..a595ef4513 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,8 +40,28 @@ Adding a large new feature or/and component to Thanos should be done by first cr * It is strongly recommended that you use OSX or popular Linux distributions systems e.g. Ubuntu, Redhat, or OpenSUSE for development. -## Pull Request Process +## Components Naming + +Thanos is a distributed system composed with several services and CLI tools as listed [here](cmd/thanos). + +When we refer to them as technical reference we use verb form: `store`, `compact`, `rule`, `query`. This includes: + +* Code +* Metrics +* Commands +* Mixin examples: alerts, rules, dashboards +* Container names +* Flags and configuration +* Package names +* Log messages, traces +However, when speaking about those or explaining we use `actor` noun form: `store gateway, compactor, ruler, querier`. This includes areas like: + +* Public communication +* Documentation +* Code comments + +## Pull Request Process 1. Read [getting started docs](docs/getting-started.md) and prepare Thanos. 2. Familiarize yourself with Go standard style guides Thanos follows: [this](https://golang.org/doc/effective_go.html) and [this](https://github.com/golang/go/wiki/CodeReviewComments). From 56a1fb6c084f11638f6f7b6532adc07dd05824b6 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 3 Feb 2020 17:12:48 +0000 Subject: [PATCH 221/257] store: Added benchmarks for .Series and expanded matchers. (#2070) ## Changes * Added ExpandedPostingsMatchers benchmarks. This function and only this will be affected by index-header changes. * Added store Series test. This can be later expanded for tests for different StoreAPIs not only Store GW. * Added few interfaces to improve testability to Store GW bucket struct. * Fixed time filtering edge case. Now request mint=0, maxt=0 should succeed. ## ExpandedPostings Script for running those: ``` #!/usr/bin/env bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd ${DIR}/../ RUN="${1}-exppostings" mkdir -p ${DIR}/bench_outs/${RUN} for TEST in BenchmarkBucketIndexReader_ExpandedPostings/binary_header BenchmarkBucketIndexReader_ExpandedPostings/index_json do echo "Running ${TEST} run ${RUN}" go test -bench=${TEST} -run=^$ -benchmem -memprofile ${DIR}/bench_outs/${RUN}/memprofile${TEST//\//-}.out -timeout 2h -benchtime 60s ./pkg/store | tee ${DIR}/bench_outs/${RUN}/bench${TEST//\//-}.out done sed 's/BenchmarkBucketIndexReader_ExpandedPostings\/binary_header/BenchmarkBucketIndexReader_ExpandedPostings/g' ${DIR}/bench_outs/${RUN}/benchBenchmarkBucketIndexReader_ExpandedPostings-binary_header.out > ${DIR}/bench_outs/${RUN}/bench_new.out sed 's/BenchmarkBucketIndexReader_ExpandedPostings\/index_json/BenchmarkBucketIndexReader_ExpandedPostings/g' ${DIR}/bench_outs/${RUN}/benchBenchmarkBucketIndexReader_ExpandedPostings-index_json.out > ${DIR}/bench_outs/${RUN}/bench_old.out benchcmp ${DIR}/bench_outs/${RUN}/bench_old.out ${DIR}/bench_outs/${RUN}/bench_new.out | tee ${DIR}/bench_outs/${RUN}/bench.out ``` Results: ``` benchmark old ns/op new ns/op delta BenchmarkBucketIndexReader_ExpandedPostings/n="1"-12 3842599 4396053 +14.40% BenchmarkBucketIndexReader_ExpandedPostings/n="1",j="foo"-12 32690723 31569625 -3.43% BenchmarkBucketIndexReader_ExpandedPostings/j="foo",n="1"-12 32505999 33343759 +2.58% BenchmarkBucketIndexReader_ExpandedPostings/n="1",j!="foo"-12 200367800 229696038 +14.64% BenchmarkBucketIndexReader_ExpandedPostings/i=~".*"-12 153160181 188532296 +23.09% BenchmarkBucketIndexReader_ExpandedPostings/i=~".+"-12 380726946 463532800 +21.75% BenchmarkBucketIndexReader_ExpandedPostings/i=~""-12 421439451 512133098 +21.52% BenchmarkBucketIndexReader_ExpandedPostings/i!=""-12 308473739 407513630 +32.11% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".*",j="foo"-12 136999349 163355090 +19.24% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".*",i!="2",j="foo"-12 161095504 178444900 +10.77% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i!=""-12 123643237 154754514 +25.16% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i!="",j="foo"-12 141086986 175977956 +24.73% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".+",j="foo"-12 212058205 244589372 +15.34% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~"1.+",j="foo"-12 33981603 46508363 +36.86% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".+",i!="2",j="foo"-12 268266004 323914937 +20.74% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".+",i!~"2.*",j="foo"-12 306779915 347659338 +13.33% benchmark old allocs new allocs delta BenchmarkBucketIndexReader_ExpandedPostings/n="1"-12 73 75 +2.74% BenchmarkBucketIndexReader_ExpandedPostings/n="1",j="foo"-12 108 112 +3.70% BenchmarkBucketIndexReader_ExpandedPostings/j="foo",n="1"-12 108 112 +3.70% BenchmarkBucketIndexReader_ExpandedPostings/n="1",j!="foo"-12 142 148 +4.23% BenchmarkBucketIndexReader_ExpandedPostings/i=~".*"-12 96 98 +2.08% BenchmarkBucketIndexReader_ExpandedPostings/i=~".+"-12 100159 300159 +199.68% BenchmarkBucketIndexReader_ExpandedPostings/i=~""-12 100114 300115 +199.77% BenchmarkBucketIndexReader_ExpandedPostings/i!=""-12 100156 300156 +199.69% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".*",j="foo"-12 149 155 +4.03% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".*",i!="2",j="foo"-12 181 193 +6.63% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i!=""-12 100174 300176 +199.65% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i!="",j="foo"-12 100204 300208 +199.60% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".+",j="foo"-12 100207 300211 +199.59% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~"1.+",j="foo"-12 11285 33511 +196.95% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".+",i!="2",j="foo"-12 100240 300247 +199.53% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".+",i!~"2.*",j="foo"-12 111349 333577 +199.58% benchmark old bytes new bytes delta BenchmarkBucketIndexReader_ExpandedPostings/n="1"-12 11379240 11379278 +0.00% BenchmarkBucketIndexReader_ExpandedPostings/n="1",j="foo"-12 23527225 23527336 +0.00% BenchmarkBucketIndexReader_ExpandedPostings/j="foo",n="1"-12 23527224 23527339 +0.00% BenchmarkBucketIndexReader_ExpandedPostings/n="1",j!="foo"-12 90635328 90636435 +0.00% BenchmarkBucketIndexReader_ExpandedPostings/i=~".*"-12 285362375 334956762 +17.38% BenchmarkBucketIndexReader_ExpandedPostings/i=~".+"-12 331578558 384372836 +15.92% BenchmarkBucketIndexReader_ExpandedPostings/i=~""-12 182041232 234834884 +29.00% BenchmarkBucketIndexReader_ExpandedPostings/i!=""-12 331576878 384371194 +15.92% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".*",j="foo"-12 92242749 141837796 +53.77% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".*",i!="2",j="foo"-12 94340164 143955415 +52.59% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i!=""-12 126308942 179103370 +41.80% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i!="",j="foo"-12 138456800 191251255 +38.13% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".+",j="foo"-12 138458444 191253140 +38.13% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~"1.+",j="foo"-12 33852975 83803000 +147.55% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".+",i!="2",j="foo"-12 207664951 260478496 +25.43% BenchmarkBucketIndexReader_ExpandedPostings/n="1",i=~".+",i!~"2.*",j="foo"-12 212691177 315435644 +48.31% ``` ## Series Script: ``` #!/usr/bin/env bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd ${DIR}/../ RUN="${1}-series" mkdir -p ${DIR}/bench_outs/${RUN} for TEST in BenchmarkSeries/binary_header BenchmarkSeries/index_json do echo "Running ${TEST} run ${RUN}" go test -bench=${TEST} -run=^$ -benchmem -memprofile ${DIR}/bench_outs/${RUN}/memprofile${TEST//\//-}.out -timeout 2h -benchtime 30s ./pkg/store | tee ${DIR}/bench_outs/${RUN}/bench${TEST//\//-}.out done sed 's/BenchmarkSeries\/binary_header/BenchmarkSeries/g' ${DIR}/bench_outs/${RUN}/benchBenchmarkSeries-binary_header.out > ${DIR}/bench_outs/${RUN}/bench_new.out sed 's/BenchmarkSeries\/index_json/BenchmarkSeries/g' ${DIR}/bench_outs/${RUN}/benchBenchmarkSeries-index_json.out > ${DIR}/bench_outs/${RUN}/bench_old.out benchcmp ${DIR}/bench_outs/${RUN}/bench_old.out ${DIR}/bench_outs/${RUN}/bench_new.out | tee ${DIR}/bench_outs/${RUN}/bench.out ``` Results: ``` Running BenchmarkSeries/binary_header run 0-series goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/store BenchmarkSeries/binary_header/1of10000000-12 10 3405980704 ns/op 1577961956 B/op 20076939 allocs/op BenchmarkSeries/binary_header/10of10000000-12 10 3338306252 ns/op 1577902473 B/op 20076885 allocs/op BenchmarkSeries/binary_header/100of10000000-12 10 3381093365 ns/op 1577926803 B/op 20077301 allocs/op BenchmarkSeries/binary_header/1000of10000000-12 10 3346731304 ns/op 1578388380 B/op 20081755 allocs/op BenchmarkSeries/binary_header/10000of10000000-12 9 3387770880 ns/op 1587132363 B/op 20127273 allocs/op BenchmarkSeries/binary_header/100000of10000000-12 10 3509686626 ns/op 1671451827 B/op 20580914 allocs/op BenchmarkSeries/binary_header/1000000of10000000-12 8 4379223614 ns/op 2514264246 B/op 25115266 allocs/op BenchmarkSeries/binary_header/10000000of10000000-12 3 12498205945 ns/op 15882149677 B/op 130614810 allocs/op PASS ok github.com/thanos-io/thanos/pkg/store 481.055s Running BenchmarkSeries/index_json run 0-series goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/store BenchmarkSeries/index_json/1of10000000-12 13 2519404503 ns/op 1437956498 B/op 15076920 allocs/op BenchmarkSeries/index_json/10of10000000-12 14 2570713484 ns/op 1437918778 B/op 15076972 allocs/op BenchmarkSeries/index_json/100of10000000-12 14 2462235340 ns/op 1437948738 B/op 15077374 allocs/op BenchmarkSeries/index_json/1000of10000000-12 15 2409482348 ns/op 1438429929 B/op 15081921 allocs/op BenchmarkSeries/index_json/10000of10000000-12 15 2447382076 ns/op 1447084683 B/op 15127167 allocs/op BenchmarkSeries/index_json/100000of10000000-12 14 2452156730 ns/op 1530657506 B/op 15580889 allocs/op BenchmarkSeries/index_json/1000000of10000000-12 10 3234136594 ns/op 2379163824 B/op 20115223 allocs/op BenchmarkSeries/index_json/10000000of10000000-12 signal: killed FAIL github.com/thanos-io/thanos/pkg/store 569.329s FAIL benchmark old ns/op new ns/op delta BenchmarkSeries/1of10000000-12 2519404503 3405980704 +35.19% BenchmarkSeries/10of10000000-12 2570713484 3338306252 +29.86% BenchmarkSeries/100of10000000-12 2462235340 3381093365 +37.32% BenchmarkSeries/1000of10000000-12 2409482348 3346731304 +38.90% BenchmarkSeries/10000of10000000-12 2447382076 3387770880 +38.42% BenchmarkSeries/100000of10000000-12 2452156730 3509686626 +43.13% BenchmarkSeries/1000000of10000000-12 3234136594 4379223614 +35.41% benchmark old allocs new allocs delta BenchmarkSeries/1of10000000-12 15076920 20076939 +33.16% BenchmarkSeries/10of10000000-12 15076972 20076885 +33.16% BenchmarkSeries/100of10000000-12 15077374 20077301 +33.16% BenchmarkSeries/1000of10000000-12 15081921 20081755 +33.15% BenchmarkSeries/10000of10000000-12 15127167 20127273 +33.05% BenchmarkSeries/100000of10000000-12 15580889 20580914 +32.09% BenchmarkSeries/1000000of10000000-12 20115223 25115266 +24.86% benchmark old bytes new bytes delta BenchmarkSeries/1of10000000-12 1437956498 1577961956 +9.74% BenchmarkSeries/10of10000000-12 1437918778 1577902473 +9.74% BenchmarkSeries/100of10000000-12 1437948738 1577926803 +9.73% BenchmarkSeries/1000of10000000-12 1438429929 1578388380 +9.73% BenchmarkSeries/10000of10000000-12 1447084683 1587132363 +9.68% BenchmarkSeries/100000of10000000-12 1530657506 1671451827 +9.20% BenchmarkSeries/1000000of10000000-12 2379163824 2514264246 +5.68% ``` Signed-off-by: Bartlomiej Plotka --- pkg/block/indexheader/binary_reader.go | 8 + pkg/block/indexheader/json_reader.go | 5 + pkg/gate/gate.go | 5 + pkg/pool/pool.go | 17 +- pkg/pool/pool_test.go | 4 +- pkg/query/api/v1_test.go | 3 +- pkg/store/bucket.go | 85 ++--- pkg/store/bucket_test.go | 462 ++++++++++++++++++++++++- pkg/store/cache/memcached_test.go | 1 - pkg/store/limiter.go | 4 + pkg/testutil/e2eutil/prometheus.go | 6 +- 11 files changed, 527 insertions(+), 73 deletions(-) diff --git a/pkg/block/indexheader/binary_reader.go b/pkg/block/indexheader/binary_reader.go index 3466046371..9005cd443b 100644 --- a/pkg/block/indexheader/binary_reader.go +++ b/pkg/block/indexheader/binary_reader.go @@ -235,9 +235,17 @@ func newBinaryWriter(fn string, buf []byte) (w *binaryWriter, err error) { dir := filepath.Dir(fn) df, err := fileutil.OpenDir(dir) + if os.IsNotExist(err) { + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return nil, err + } + df, err = fileutil.OpenDir(dir) + } if err != nil { + return nil, err } + defer runutil.CloseWithErrCapture(&err, df, "dir close") if err := os.RemoveAll(fn); err != nil { diff --git a/pkg/block/indexheader/json_reader.go b/pkg/block/indexheader/json_reader.go index d8ef13193f..5a3654f4d6 100644 --- a/pkg/block/indexheader/json_reader.go +++ b/pkg/block/indexheader/json_reader.go @@ -210,6 +210,11 @@ func NewJSONReader(ctx context.Context, logger log.Logger, bkt objstore.BucketRe return nil, errors.Wrap(err, "read index cache") } + // Just in case the dir was not created. + if err := os.MkdirAll(filepath.Join(dir, id.String()), os.ModePerm); err != nil { + return nil, errors.Wrap(err, "create dir") + } + // Try to download index cache file from object store. if err = objstore.DownloadFile(ctx, logger, bkt, filepath.Join(id.String(), block.IndexCacheFilename), cachefn); err == nil { return newFileJSONReader(logger, cachefn) diff --git a/pkg/gate/gate.go b/pkg/gate/gate.go index 6b7dbfc448..43bc3a47b4 100644 --- a/pkg/gate/gate.go +++ b/pkg/gate/gate.go @@ -11,6 +11,11 @@ import ( "github.com/prometheus/prometheus/pkg/gate" ) +type Gater interface { + IsMyTurn(ctx context.Context) error + Done() +} + // Gate wraps the Prometheus gate with extra metrics. type Gate struct { g *gate.Gate diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go index 939628acbf..ede9e667bf 100644 --- a/pkg/pool/pool.go +++ b/pkg/pool/pool.go @@ -9,10 +9,15 @@ import ( "github.com/pkg/errors" ) -// BytesPool is a bucketed pool for variably sized byte slices. It can be configured to not allow +type BytesPool interface { + Get(sz int) (*[]byte, error) + Put(b *[]byte) +} + +// BucketedBytesPool is a bucketed pool for variably sized byte slices. It can be configured to not allow // more than a maximum number of bytes being used at a given time. // Every byte slice obtained from the pool must be returned. -type BytesPool struct { +type BucketedBytesPool struct { buckets []sync.Pool sizes []int maxTotal uint64 @@ -25,7 +30,7 @@ type BytesPool struct { // NewBytesPool returns a new BytesPool with size buckets for minSize to maxSize // increasing by the given factor and maximum number of used bytes. // No more than maxTotal bytes can be used at any given time unless maxTotal is set to 0. -func NewBytesPool(minSize, maxSize int, factor float64, maxTotal uint64) (*BytesPool, error) { +func NewBucketedBytesPool(minSize, maxSize int, factor float64, maxTotal uint64) (*BucketedBytesPool, error) { if minSize < 1 { return nil, errors.New("invalid minimum pool size") } @@ -41,7 +46,7 @@ func NewBytesPool(minSize, maxSize int, factor float64, maxTotal uint64) (*Bytes for s := minSize; s <= maxSize; s = int(float64(s) * factor) { sizes = append(sizes, s) } - p := &BytesPool{ + p := &BucketedBytesPool{ buckets: make([]sync.Pool, len(sizes)), sizes: sizes, maxTotal: maxTotal, @@ -57,7 +62,7 @@ func NewBytesPool(minSize, maxSize int, factor float64, maxTotal uint64) (*Bytes var ErrPoolExhausted = errors.New("pool exhausted") // Get returns a new byte slices that fits the given size. -func (p *BytesPool) Get(sz int) (*[]byte, error) { +func (p *BucketedBytesPool) Get(sz int) (*[]byte, error) { p.mtx.Lock() defer p.mtx.Unlock() @@ -84,7 +89,7 @@ func (p *BytesPool) Get(sz int) (*[]byte, error) { } // Put returns a byte slice to the right bucket in the pool. -func (p *BytesPool) Put(b *[]byte) { +func (p *BucketedBytesPool) Put(b *[]byte) { if b == nil { return } diff --git a/pkg/pool/pool_test.go b/pkg/pool/pool_test.go index 955e85b08b..8bf3c302c3 100644 --- a/pkg/pool/pool_test.go +++ b/pkg/pool/pool_test.go @@ -16,7 +16,7 @@ import ( ) func TestBytesPool(t *testing.T) { - chunkPool, err := NewBytesPool(10, 100, 2, 1000) + chunkPool, err := NewBucketedBytesPool(10, 100, 2, 1000) testutil.Ok(t, err) testutil.Equals(t, []int{10, 20, 40, 80}, chunkPool.sizes) @@ -61,7 +61,7 @@ func TestBytesPool(t *testing.T) { } func TestRacePutGet(t *testing.T) { - chunkPool, err := NewBytesPool(3, 100, 2, 5000) + chunkPool, err := NewBucketedBytesPool(3, 100, 2, 5000) testutil.Ok(t, err) defer leaktest.CheckTimeout(t, 10*time.Second)() diff --git a/pkg/query/api/v1_test.go b/pkg/query/api/v1_test.go index 481754f3ac..c85fbe9e66 100644 --- a/pkg/query/api/v1_test.go +++ b/pkg/query/api/v1_test.go @@ -1080,9 +1080,8 @@ func BenchmarkQueryResultEncoding(b *testing.B) { } b.ResetTimer() - c, err := json.Marshal(&input) + _, err := json.Marshal(&input) testutil.Ok(b, err) - fmt.Println(len(c)) } func TestParseDownsamplingParamMillis(t *testing.T) { diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 8b09feafc9..e7de6cdc40 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -74,6 +74,8 @@ const ( // This label name is intentionally against Prometheus label style. // TODO(bwplotka): Remove it at some point. CompatibilityTypeLabelName = "@thanos_compatibility_store_type" + + partitionerMaxGapSize = 512 * 1024 ) type bucketStoreMetrics struct { @@ -216,7 +218,7 @@ type BucketStore struct { fetcher block.MetadataFetcher dir string indexCache storecache.IndexCache - chunkPool *pool.BytesPool + chunkPool pool.BytesPool // Sets of blocks that have the same labels. They are indexed by a hash over their label set. mtx sync.RWMutex @@ -229,10 +231,10 @@ type BucketStore struct { blockSyncConcurrency int // Query gate which limits the maximum amount of concurrent queries. - queryGate *gate.Gate + queryGate gate.Gater // samplesLimiter limits the number of samples per each Series() call. - samplesLimiter *Limiter + samplesLimiter SampleLimiter partitioner partitioner filterConfig *FilterConfig @@ -267,13 +269,11 @@ func NewBucketStore( return nil, errors.Errorf("max concurrency value cannot be lower than 0 (got %v)", maxConcurrent) } - chunkPool, err := pool.NewBytesPool(maxChunkSize, 50e6, 2, maxChunkPoolBytes) + chunkPool, err := pool.NewBucketedBytesPool(maxChunkSize, 50e6, 2, maxChunkPoolBytes) if err != nil { return nil, errors.Wrap(err, "create chunk pool") } - const maxGapSize = 512 * 1024 - metrics := newBucketStoreMetrics(reg) s := &BucketStore{ logger: logger, @@ -292,7 +292,7 @@ func NewBucketStore( extprom.WrapRegistererWithPrefix("thanos_bucket_store_series_", reg), ), samplesLimiter: NewLimiter(maxSampleCount, metrics.queriesDropped), - partitioner: gapBasedPartitioner{maxGapSize: maxGapSize}, + partitioner: gapBasedPartitioner{maxGapSize: partitionerMaxGapSize}, enableCompatibilityLabel: enableCompatibilityLabel, enableIndexHeader: enableIndexHeader, } @@ -433,10 +433,6 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er dir := filepath.Join(s.dir, meta.ULID.String()) start := time.Now() - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return errors.Wrap(err, "create dir") - } - level.Debug(s.logger).Log("msg", "loading new block", "id", meta.ULID) defer func() { if err != nil { @@ -570,6 +566,10 @@ func (s *BucketStore) Info(context.Context, *storepb.InfoRequest) (*storepb.Info } func (s *BucketStore) limitMinTime(mint int64) int64 { + if s.filterConfig == nil { + return mint + } + filterMinTime := s.filterConfig.MinTime.PrometheusTimestamp() if mint < filterMinTime { @@ -580,6 +580,10 @@ func (s *BucketStore) limitMinTime(mint int64) int64 { } func (s *BucketStore) limitMaxTime(maxt int64) int64 { + if s.filterConfig == nil { + return maxt + } + filterMaxTime := s.filterConfig.MaxTime.PrometheusTimestamp() if maxt > filterMaxTime { @@ -630,7 +634,7 @@ func blockSeries( chunkr *bucketChunkReader, matchers []*labels.Matcher, req *storepb.SeriesRequest, - samplesLimiter *Limiter, + samplesLimiter SampleLimiter, ) (storepb.SeriesSet, *queryStats, error) { ps, err := indexr.ExpandedPostings(matchers) if err != nil { @@ -844,6 +848,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie if !ok { continue } + blocks := bs.getFor(req.MinTime, req.MaxTime, req.MaxResolutionWindow) mtx.Lock() @@ -855,7 +860,6 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie } for _, b := range blocks { - b := b // We must keep the readers open until all their data has been sent. @@ -1109,7 +1113,7 @@ func int64index(s []int64, x int64) int { // getFor returns a time-ordered list of blocks that cover date between mint and maxt. // Blocks with the biggest resolution possible but not bigger than the given max resolution are returned. func (s *bucketBlockSet) getFor(mint, maxt, maxResolutionMillis int64) (bs []*bucketBlock) { - if mint == maxt { + if mint > maxt { return nil } @@ -1122,20 +1126,23 @@ func (s *bucketBlockSet) getFor(mint, maxt, maxResolutionMillis int64) (bs []*bu } // Fill the given interval with the blocks for the current resolution. - // Our current resolution might not cover all data, so recursively fill the gaps with higher resolution blocks if there is any. + // Our current resolution might not cover all data, so recursively fill the gaps with higher resolution blocks + // if there is any. start := mint for _, b := range s.blocks[i] { if b.meta.MaxTime <= mint { continue } - if b.meta.MinTime >= maxt { + // NOTE: Block intervals are half-open: [b.MinTime, b.MaxTime). + if b.meta.MinTime > maxt { break } if i+1 < len(s.resolutions) { - bs = append(bs, s.getFor(start, b.meta.MinTime, s.resolutions[i+1])...) + bs = append(bs, s.getFor(start, b.meta.MinTime-1, s.resolutions[i+1])...) } bs = append(bs, b) + start = b.meta.MaxTime } @@ -1167,11 +1174,11 @@ func (s *bucketBlockSet) labelMatchers(matchers ...*labels.Matcher) ([]*labels.M // state for the block on local disk. type bucketBlock struct { logger log.Logger - bucket objstore.BucketReader + bkt objstore.BucketReader meta *metadata.Meta dir string indexCache storecache.IndexCache - chunkPool *pool.BytesPool + chunkPool pool.BytesPool indexHeaderReader indexheader.Reader @@ -1191,14 +1198,14 @@ func newBucketBlock( bkt objstore.BucketReader, dir string, indexCache storecache.IndexCache, - chunkPool *pool.BytesPool, + chunkPool pool.BytesPool, indexHeadReader indexheader.Reader, p partitioner, seriesRefetches prometheus.Counter, ) (b *bucketBlock, err error) { b = &bucketBlock{ logger: logger, - bucket: bkt, + bkt: bkt, indexCache: indexCache, chunkPool: chunkPool, dir: dir, @@ -1224,7 +1231,7 @@ func (b *bucketBlock) indexFilename() string { } func (b *bucketBlock) readIndexRange(ctx context.Context, off, length int64) ([]byte, error) { - r, err := b.bucket.GetRange(ctx, b.indexFilename(), off, length) + r, err := b.bkt.GetRange(ctx, b.indexFilename(), off, length) if err != nil { return nil, errors.Wrap(err, "get range reader") } @@ -1244,7 +1251,7 @@ func (b *bucketBlock) readChunkRange(ctx context.Context, seq int, off, length i } buf := bytes.NewBuffer(*c) - r, err := b.bucket.GetRange(ctx, b.chunkObjs[seq], off, length) + r, err := b.bkt.GetRange(ctx, b.chunkObjs[seq], off, length) if err != nil { b.chunkPool.Put(c) return nil, errors.Wrap(err, "get range reader") @@ -1261,7 +1268,7 @@ func (b *bucketBlock) readChunkRange(ctx context.Context, seq int, off, length i func (b *bucketBlock) indexReader(ctx context.Context) *bucketIndexReader { b.pendingReaders.Add(1) - return newBucketIndexReader(ctx, b.logger, b, b.indexCache) + return newBucketIndexReader(ctx, b) } func (b *bucketBlock) chunkReader(ctx context.Context) *bucketChunkReader { @@ -1278,27 +1285,23 @@ func (b *bucketBlock) Close() error { // bucketIndexReader is a custom index reader (not conforming index.Reader interface) that reads index that is stored in // object storage without having to fully download it. type bucketIndexReader struct { - logger log.Logger - ctx context.Context - block *bucketBlock - dec *index.Decoder - stats *queryStats - cache storecache.IndexCache + ctx context.Context + block *bucketBlock + dec *index.Decoder + stats *queryStats mtx sync.Mutex loadedSeries map[uint64][]byte } -func newBucketIndexReader(ctx context.Context, logger log.Logger, block *bucketBlock, cache storecache.IndexCache) *bucketIndexReader { +func newBucketIndexReader(ctx context.Context, block *bucketBlock) *bucketIndexReader { r := &bucketIndexReader{ - logger: logger, - ctx: ctx, - block: block, + ctx: ctx, + block: block, dec: &index.Decoder{ LookupSymbol: block.indexHeaderReader.LookupSymbol, }, stats: &queryStats{}, - cache: cache, loadedSeries: map[uint64][]byte{}, } return r @@ -1464,7 +1467,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { keys = append(keys, g.keys...) } - fromCache, _ := r.cache.FetchMultiPostings(r.ctx, r.block.meta.ULID, keys) + fromCache, _ := r.block.indexCache.FetchMultiPostings(r.ctx, r.block.meta.ULID, keys) // Iterate over all groups and fetch posting from cache. // If we have a miss, mark key to be fetched in `ptrs` slice. @@ -1548,7 +1551,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { // Return postings and fill LRU cache. // Truncate first 4 bytes which are length of posting. groups[p.groupID].Fill(p.keyID, newBigEndianPostings(pBytes[4:])) - r.cache.StorePostings(r.ctx, r.block.meta.ULID, groups[p.groupID].keys[p.keyID], pBytes) + r.block.indexCache.StorePostings(r.ctx, r.block.meta.ULID, groups[p.groupID].keys[p.keyID], pBytes) // If we just fetched it we still have to update the stats for touched postings. r.stats.postingsTouched++ @@ -1629,7 +1632,7 @@ func (it *bigEndianPostings) Err() error { func (r *bucketIndexReader) PreloadSeries(ids []uint64) error { // Load series from cache, overwriting the list of ids to preload // with the missing ones. - fromCache, ids := r.cache.FetchMultiSeries(r.ctx, r.block.meta.ULID, ids) + fromCache, ids := r.block.indexCache.FetchMultiSeries(r.ctx, r.block.meta.ULID, ids) for id, b := range fromCache { r.loadedSeries[id] = b } @@ -1678,7 +1681,7 @@ func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, refetc // Inefficient, but should be rare. r.block.seriesRefetches.Inc() - level.Warn(r.logger).Log("msg", "series size exceeded expected size; refetching", "id", id, "series length", n+int(l), "maxSeriesSize", maxSeriesSize) + level.Warn(r.block.logger).Log("msg", "series size exceeded expected size; refetching", "id", id, "series length", n+int(l), "maxSeriesSize", maxSeriesSize) // Fetch plus to get the size of next one if exists. return r.loadSeries(ctx, ids[i:], true, id, id+uint64(n+int(l)+1)) @@ -1686,7 +1689,7 @@ func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, refetc c = c[n : n+int(l)] r.mtx.Lock() r.loadedSeries[id] = c - r.cache.StoreSeries(r.ctx, r.block.meta.ULID, id, c) + r.block.indexCache.StoreSeries(r.ctx, r.block.meta.ULID, id, c) r.mtx.Unlock() } return nil @@ -1800,7 +1803,7 @@ func (r *bucketChunkReader) addPreload(id uint64) error { } // preload all added chunk IDs. Must be called before the first call to Chunk is made. -func (r *bucketChunkReader) preload(samplesLimiter *Limiter) error { +func (r *bucketChunkReader) preload(samplesLimiter SampleLimiter) error { g, ctx := errgroup.WithContext(r.ctx) numChunks := uint64(0) diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index a277f392cf..2fa3b2f089 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -6,6 +6,7 @@ package store import ( "bytes" "context" + "fmt" "io" "io/ioutil" "math" @@ -13,7 +14,9 @@ import ( "path" "path/filepath" "sort" + "strconv" "sync" + "sync/atomic" "testing" "time" @@ -27,12 +30,16 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/encoding" "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/indexheader" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/objstore/filesystem" "github.com/thanos-io/thanos/pkg/objstore/inmem" + "github.com/thanos-io/thanos/pkg/pool" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" "github.com/thanos-io/thanos/pkg/testutil/e2eutil" @@ -206,7 +213,7 @@ func TestBucketBlockSet_addGet(t *testing.T) { testutil.Ok(t, set.add(&bucketBlock{meta: &m})) } - cases := []struct { + for _, c := range []struct { mint, maxt int64 maxResolution int64 res []resBlock @@ -230,6 +237,8 @@ func TestBucketBlockSet_addGet(t *testing.T) { {window: downsample.ResLevel0, mint: 100, maxt: 200}, {window: downsample.ResLevel0, mint: 200, maxt: 300}, {window: downsample.ResLevel0, mint: 300, maxt: 400}, + // Block intervals are half-open: [b.MinTime, b.MaxTime), so 400-500 contains single sample. + {window: downsample.ResLevel0, mint: 400, maxt: 500}, }, }, { mint: 100, @@ -253,20 +262,18 @@ func TestBucketBlockSet_addGet(t *testing.T) { {window: downsample.ResLevel0, mint: 400, maxt: 500}, }, }, - } - for i, c := range cases { - t.Logf("case %d", i) - - var exp []*bucketBlock - for _, b := range c.res { - var m metadata.Meta - m.Thanos.Downsample.Resolution = b.window - m.MinTime = b.mint - m.MaxTime = b.maxt - exp = append(exp, &bucketBlock{meta: &m}) - } - res := set.getFor(c.mint, c.maxt, c.maxResolution) - testutil.Equals(t, exp, res) + } { + t.Run("", func(t *testing.T) { + var exp []*bucketBlock + for _, b := range c.res { + var m metadata.Meta + m.Thanos.Downsample.Resolution = b.window + m.MinTime = b.mint + m.MaxTime = b.maxt + exp = append(exp, &bucketBlock{meta: &m}) + } + testutil.Equals(t, exp, set.getFor(c.mint, c.maxt, c.maxResolution)) + }) } } @@ -791,11 +798,14 @@ func TestReadIndexCache_LoadSeries(t *testing.T) { s := newBucketStoreMetrics(nil) b := &bucketBlock{ meta: &metadata.Meta{ - BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil)}, + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(1, nil), + }, }, - bucket: bkt, + bkt: bkt, seriesRefetches: s.seriesRefetches, logger: log.NewNopLogger(), + indexCache: noopCache{}, } buf := encoding.Encbuf{} @@ -813,8 +823,6 @@ func TestReadIndexCache_LoadSeries(t *testing.T) { block: b, stats: &queryStats{}, loadedSeries: map[uint64][]byte{}, - cache: noopCache{}, - logger: log.NewNopLogger(), } // Success with no refetches. @@ -854,3 +862,419 @@ func TestReadIndexCache_LoadSeries(t *testing.T) { // Fail, but no recursion at least. testutil.NotOk(t, r.loadSeries(context.TODO(), []uint64{2, 13, 24}, false, 1, 15)) } + +func TestBucketIndexReader_ExpandedPostings(t *testing.T) { + t.Skip("") + tb := testutil.NewTB(t) + + tmpDir, err := ioutil.TempDir("", "test-expanded-postings") + testutil.Ok(tb, err) + defer func() { testutil.Ok(tb, os.RemoveAll(tmpDir)) }() + + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(tb, err) + defer func() { testutil.Ok(tb, bkt.Close()) }() + + id := uploadTestBlock(tb, tmpDir, bkt) + + tb.Run("binary_header", func(tb testutil.TB) { + r, err := indexheader.NewBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, id) + testutil.Ok(tb, err) + + benchmarkExpandedPostings(tb, bkt, id, r) + }) + tb.Run("index_json", func(tb testutil.TB) { + r, err := indexheader.NewJSONReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, id) + testutil.Ok(tb, err) + + benchmarkExpandedPostings(tb, bkt, id, r) + }) +} + +func BenchmarkBucketIndexReader_ExpandedPostings(b *testing.B) { + tb := testutil.NewTB(b) + + tmpDir, err := ioutil.TempDir("", "bench-expanded-postings") + testutil.Ok(tb, err) + defer func() { testutil.Ok(tb, os.RemoveAll(tmpDir)) }() + + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(tb, err) + defer func() { testutil.Ok(tb, bkt.Close()) }() + + id := uploadTestBlock(tb, tmpDir, bkt) + tb.Run("binary_header", func(tb testutil.TB) { + r, err := indexheader.NewBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, id) + testutil.Ok(tb, err) + + benchmarkExpandedPostings(tb, bkt, id, r) + }) + tb.Run("index_json", func(tb testutil.TB) { + r, err := indexheader.NewJSONReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, id) + testutil.Ok(tb, err) + + benchmarkExpandedPostings(tb, bkt, id, r) + }) +} + +// Make entries ~50B in size, to emulate real-world high cardinality. +const ( + postingsBenchSuffix = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd" +) + +func uploadTestBlock(t testing.TB, tmpDir string, bkt objstore.Bucket) ulid.ULID { + h, err := tsdb.NewHead(nil, nil, nil, 1000) + testutil.Ok(t, err) + defer func() { + testutil.Ok(t, h.Close()) + }() + + logger := log.NewNopLogger() + + app := h.Appender() + addSeries := func(l labels.Labels) { + _, err := app.Add(l, 0, 0) + testutil.Ok(t, err) + } + + for n := 0; n < 10; n++ { + for i := 0; i < 100000; i++ { + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", strconv.Itoa(n)+postingsBenchSuffix, "j", "foo")) + // Have some series that won't be matched, to properly test inverted matches. + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", strconv.Itoa(n)+postingsBenchSuffix, "j", "bar")) + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", "0_"+strconv.Itoa(n)+postingsBenchSuffix, "j", "bar")) + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", "1_"+strconv.Itoa(n)+postingsBenchSuffix, "j", "bar")) + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", "2_"+strconv.Itoa(n)+postingsBenchSuffix, "j", "foo")) + } + } + testutil.Ok(t, app.Commit()) + + testutil.Ok(t, os.MkdirAll(filepath.Join(tmpDir, "tmp"), os.ModePerm)) + id := createBlockFromHead(t, filepath.Join(tmpDir, "tmp"), h) + + _, err = metadata.InjectThanos(log.NewNopLogger(), filepath.Join(tmpDir, "tmp", id.String()), metadata.Thanos{ + Labels: labels.Labels{{Name: "ext1", Value: "1"}}.Map(), + Downsample: metadata.ThanosDownsample{Resolution: 0}, + Source: metadata.TestSource, + }, nil) + testutil.Ok(t, err) + testutil.Ok(t, block.Upload(context.Background(), logger, bkt, filepath.Join(tmpDir, "tmp", id.String()))) + + return id +} + +func createBlockFromHead(t testing.TB, dir string, head *tsdb.Head) ulid.ULID { + compactor, err := tsdb.NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil) + testutil.Ok(t, err) + + testutil.Ok(t, os.MkdirAll(dir, 0777)) + + // Add +1 millisecond to block maxt because block intervals are half-open: [b.MinTime, b.MaxTime). + // Because of this block intervals are always +1 than the total samples it includes. + ulid, err := compactor.Write(dir, head, head.MinTime(), head.MaxTime()+1, nil) + testutil.Ok(t, err) + return ulid +} + +// Very similar benchmark to ths: https://github.com/prometheus/prometheus/blob/1d1732bc25cc4b47f513cb98009a4eb91879f175/tsdb/querier_bench_test.go#L82, +// but with postings results check when run as test. +func benchmarkExpandedPostings( + t testutil.TB, + bkt objstore.BucketReader, + id ulid.ULID, + r indexheader.Reader, +) { + n1 := labels.MustNewMatcher(labels.MatchEqual, "n", "1"+postingsBenchSuffix) + + jFoo := labels.MustNewMatcher(labels.MatchEqual, "j", "foo") + jNotFoo := labels.MustNewMatcher(labels.MatchNotEqual, "j", "foo") + + iStar := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.*$") + iPlus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.+$") + i1Plus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^1.+$") + iEmptyRe := labels.MustNewMatcher(labels.MatchRegexp, "i", "^$") + iNotEmpty := labels.MustNewMatcher(labels.MatchNotEqual, "i", "") + iNot2 := labels.MustNewMatcher(labels.MatchNotEqual, "n", "2"+postingsBenchSuffix) + iNot2Star := labels.MustNewMatcher(labels.MatchNotRegexp, "i", "^2.*$") + + cases := []struct { + name string + matchers []*labels.Matcher + + expectedLen int + }{ + {`n="1"`, []*labels.Matcher{n1}, 200000}, + {`n="1",j="foo"`, []*labels.Matcher{n1, jFoo}, 100000}, + {`j="foo",n="1"`, []*labels.Matcher{jFoo, n1}, 100000}, + {`n="1",j!="foo"`, []*labels.Matcher{n1, jNotFoo}, 100000}, + {`i=~".*"`, []*labels.Matcher{iStar}, 5000000}, + {`i=~".+"`, []*labels.Matcher{iPlus}, 5000000}, + {`i=~""`, []*labels.Matcher{iEmptyRe}, 0}, + {`i!=""`, []*labels.Matcher{iNotEmpty}, 5000000}, + {`n="1",i=~".*",j="foo"`, []*labels.Matcher{n1, iStar, jFoo}, 100000}, + {`n="1",i=~".*",i!="2",j="foo"`, []*labels.Matcher{n1, iStar, iNot2, jFoo}, 100000}, + {`n="1",i!=""`, []*labels.Matcher{n1, iNotEmpty}, 200000}, + {`n="1",i!="",j="foo"`, []*labels.Matcher{n1, iNotEmpty, jFoo}, 100000}, + {`n="1",i=~".+",j="foo"`, []*labels.Matcher{n1, iPlus, jFoo}, 100000}, + {`n="1",i=~"1.+",j="foo"`, []*labels.Matcher{n1, i1Plus, jFoo}, 11111}, + {`n="1",i=~".+",i!="2",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2, jFoo}, 100000}, + {`n="1",i=~".+",i!~"2.*",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2Star, jFoo}, 88889}, + } + + for _, c := range cases { + t.Run(c.name, func(t testutil.TB) { + b := &bucketBlock{ + logger: log.NewNopLogger(), + indexHeaderReader: r, + indexCache: noopCache{}, + bkt: bkt, + meta: &metadata.Meta{BlockMeta: tsdb.BlockMeta{ULID: id}}, + partitioner: gapBasedPartitioner{maxGapSize: partitionerMaxGapSize}, + } + + indexr := newBucketIndexReader(context.Background(), b) + + t.ResetTimer() + for i := 0; i < t.N(); i++ { + p, err := indexr.ExpandedPostings(c.matchers) + testutil.Ok(t, err) + testutil.Equals(t, c.expectedLen, len(p)) + } + }) + } +} + +func newSeries(t testing.TB, lset labels.Labels, smplChunks [][]sample) storepb.Series { + var s storepb.Series + + for _, l := range lset { + s.Labels = append(s.Labels, storepb.Label{Name: l.Name, Value: l.Value}) + } + + for _, smpls := range smplChunks { + c := chunkenc.NewXORChunk() + a, err := c.Appender() + testutil.Ok(t, err) + + for _, smpl := range smpls { + a.Append(smpl.t, smpl.v) + } + + ch := storepb.AggrChunk{ + MinTime: smpls[0].t, + MaxTime: smpls[len(smpls)-1].t, + Raw: &storepb.Chunk{Type: storepb.Chunk_XOR, Data: c.Bytes()}, + } + + s.Chunks = append(s.Chunks, ch) + } + return s +} + +func TestSeries_200kSeriesWithOneSample(t *testing.T) { + benchSeries_SeriesWithOneSample(testutil.NewTB(t), 200000) +} +func BenchmarkSeries(b *testing.B) { + benchSeries_SeriesWithOneSample(testutil.NewTB(b), 10000000) +} + +func benchSeries_SeriesWithOneSample(t testutil.TB, totalSeries int) { + tmpDir, err := ioutil.TempDir("", "testorbench-series") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() + + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, bkt.Close()) }() + + logger := log.NewNopLogger() + var blocks []*bucketBlock + + numSeriesPerBlock := totalSeries / 4 + series := make([]storepb.Series, 0, 4*numSeriesPerBlock) + + thanosMeta := metadata.Thanos{ + Labels: labels.Labels{{Name: "ext1", Value: "1"}}.Map(), + Downsample: metadata.ThanosDownsample{Resolution: 0}, + Source: metadata.TestSource, + } + + var chunkPool pool.BytesPool + chunkPool, err = pool.NewBucketedBytesPool(maxChunkSize, 50e6, 2, 100e7) + testutil.Ok(t, err) + + if !t.IsBenchmark() { + bucketedPool, err := pool.NewBucketedBytesPool(maxChunkSize, 50e6, 2, 100e7) + testutil.Ok(t, err) + + chunkPool = &mockedPool{parent: bucketedPool} + } + + // Create 4 blocks. Each will have numSeriesPerBlock number of series that have 1 sample only. + // Timestamp will be counted for each new series, so each series will have unique timestamp. + // This allows to pick time range that will correspond to number of series picked 1:1. + for bi := 0; bi < 4; bi++ { + h, err := tsdb.NewHead(nil, nil, nil, 1) + testutil.Ok(t, err) + defer testutil.Ok(t, h.Close()) + + app := h.Appender() + + for i := 0; i < numSeriesPerBlock; i++ { + ts := int64(bi*numSeriesPerBlock + i) + lbls := labels.FromStrings("foo", "bar", "i", fmt.Sprintf("%07d%s", ts, postingsBenchSuffix)) + series = append(series, newSeries(t, append(labels.Labels{{Name: "ext1", Value: "1"}}, lbls...), [][]sample{{sample{t: ts, v: 0}}})) + + _, err := app.Add(lbls, ts, 0) + testutil.Ok(t, err) + } + testutil.Ok(t, app.Commit()) + + blockDir := filepath.Join(tmpDir, "tmp") + id := createBlockFromHead(t, blockDir, h) + + meta, err := metadata.InjectThanos(log.NewNopLogger(), filepath.Join(blockDir, id.String()), thanosMeta, nil) + testutil.Ok(t, err) + testutil.Ok(t, block.Upload(context.Background(), logger, bkt, filepath.Join(blockDir, id.String()))) + + b := &bucketBlock{ + indexCache: noopCache{}, + logger: logger, + bkt: bkt, + meta: meta, + partitioner: gapBasedPartitioner{maxGapSize: partitionerMaxGapSize}, + chunkObjs: []string{filepath.Join(id.String(), "chunks", "000001")}, + chunkPool: chunkPool, + } + blocks = append(blocks, b) + } + + store := &BucketStore{ + bkt: bkt, + logger: logger, + indexCache: noopCache{}, + metrics: newBucketStoreMetrics(nil), + blockSets: map[uint64]*bucketBlockSet{ + labels.Labels{{Name: "ext1", Value: "1"}}.Hash(): {blocks: [][]*bucketBlock{blocks}}, + }, + queryGate: noopGater{}, + samplesLimiter: noopLimiter{}, + } + + var cases []*benchSeriesCase + if t.IsBenchmark() { + for s := 1; s <= 4*numSeriesPerBlock; s *= 10 { + cases = append(cases, &benchSeriesCase{ + name: fmt.Sprintf("%dof%d", s, 4*numSeriesPerBlock), + req: &storepb.SeriesRequest{ + MinTime: 0, + MaxTime: int64(s) - 1, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "foo", Value: "bar"}, + }, + }, + expected: series[:s], + }) + } + } else { + // For test just go straight away to the largest one, we care here only about correctness. + s := totalSeries + + cases = append(cases, &benchSeriesCase{ + name: fmt.Sprintf("%dof%d", s, 4*numSeriesPerBlock), + req: &storepb.SeriesRequest{ + MinTime: 0, + MaxTime: int64(s) - 1, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "foo", Value: "bar"}, + }, + }, + expected: series[:s], + }) + } + + t.Run("binary_header", func(t testutil.TB) { + for _, block := range blocks { + block.indexHeaderReader, err = indexheader.NewBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, block.meta.ULID) + testutil.Ok(t, err) + } + + benchmarkSeries(t, store, cases) + if !t.IsBenchmark() { + // Make sure pool is correctly used. + testutil.Equals(t, 4, int(chunkPool.(*mockedPool).gets)) + testutil.Equals(t, 0, int(chunkPool.(*mockedPool).balance)) + chunkPool.(*mockedPool).gets = 0 + } + }) + + t.Run("index_json", func(t testutil.TB) { + for _, block := range blocks { + block.indexHeaderReader, err = indexheader.NewJSONReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, block.meta.ULID) + testutil.Ok(t, err) + } + + benchmarkSeries(t, store, cases) + if !t.IsBenchmark() { + // Make sure pool is correctly used. + testutil.Equals(t, 4, int(chunkPool.(*mockedPool).gets)) + testutil.Equals(t, 0, int(chunkPool.(*mockedPool).balance)) + } + }) +} + +type mockedPool struct { + parent pool.BytesPool + balance uint64 + gets uint64 +} + +func (m *mockedPool) Get(sz int) (*[]byte, error) { + b, err := m.parent.Get(sz) + if err != nil { + return nil, err + } + atomic.AddUint64(&m.balance, uint64(cap(*b))) + atomic.AddUint64(&m.gets, uint64(1)) + return b, nil +} + +func (m *mockedPool) Put(b *[]byte) { + atomic.AddUint64(&m.balance, ^uint64(cap(*b)-1)) + m.parent.Put(b) +} + +type noopGater struct{} + +func (noopGater) IsMyTurn(context.Context) error { return nil } +func (noopGater) Done() {} + +type noopLimiter struct{} + +func (noopLimiter) Check(uint64) error { return nil } + +type benchSeriesCase struct { + name string + req *storepb.SeriesRequest + expected []storepb.Series +} + +func benchmarkSeries(t testutil.TB, store *BucketStore, cases []*benchSeriesCase) { + for _, c := range cases { + t.Run(c.name, func(t testutil.TB) { + t.ResetTimer() + for i := 0; i < t.N(); i++ { + srv := newStoreSeriesServer(context.Background()) + testutil.Ok(t, store.Series(c.req, srv)) + testutil.Equals(t, 0, len(srv.Warnings)) + testutil.Equals(t, len(c.expected), len(srv.SeriesSet)) + + if !t.IsBenchmark() { + // This will give unreadable output for millions of series error. + testutil.Equals(t, c.expected, srv.SeriesSet) + } + + } + }) + } +} diff --git a/pkg/store/cache/memcached_test.go b/pkg/store/cache/memcached_test.go index c44b641758..0463bbfb11 100644 --- a/pkg/store/cache/memcached_test.go +++ b/pkg/store/cache/memcached_test.go @@ -19,7 +19,6 @@ import ( func TestMemcachedIndexCache_FetchMultiPostings(t *testing.T) { t.Parallel() - defer leaktest.CheckTimeout(t, 10*time.Second)() // Init some data to conveniently define test cases later one. block1 := ulid.MustNew(1, nil) diff --git a/pkg/store/limiter.go b/pkg/store/limiter.go index fc7216549c..5c23752d73 100644 --- a/pkg/store/limiter.go +++ b/pkg/store/limiter.go @@ -8,6 +8,10 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +type SampleLimiter interface { + Check(num uint64) error +} + // Limiter is a simple mechanism for checking if something has passed a certain threshold. type Limiter struct { limit uint64 diff --git a/pkg/testutil/e2eutil/prometheus.go b/pkg/testutil/e2eutil/prometheus.go index 56972041ce..b298ae91f9 100644 --- a/pkg/testutil/e2eutil/prometheus.go +++ b/pkg/testutil/e2eutil/prometheus.go @@ -253,8 +253,10 @@ func (p *Prometheus) Stop() error { return nil } - if err := p.cmd.Process.Signal(syscall.SIGTERM); err != nil { - return errors.Wrapf(err, "failed to Prometheus. Kill it manually and clean %s dir", p.db.Dir()) + if p.cmd.Process != nil { + if err := p.cmd.Process.Signal(syscall.SIGTERM); err != nil { + return errors.Wrapf(err, "failed to Prometheus. Kill it manually and clean %s dir", p.db.Dir()) + } } time.Sleep(time.Second / 2) return p.cleanup() From 7c02430c59c2d1d2491f2c5a587e9a74e5c1a423 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Tue, 4 Feb 2020 18:29:04 +0800 Subject: [PATCH 222/257] Mixin: Update sidecar alert (#2002) * Mixin: update sidecar alert Signed-off-by: Xiang Dai <764524258@qq.com> * remove nonexistent alert Signed-off-by: Xiang Dai <764524258@qq.com> * feedback Signed-off-by: Xiang Dai <764524258@qq.com> * feedback Signed-off-by: Xiang Dai <764524258@qq.com> --- examples/alerts/alerts.md | 39 +++++++++------------------ examples/alerts/alerts.yaml | 8 ++++++ mixin/thanos/alerts/sidecar.libsonnet | 13 +++++++++ 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index b45f78d8c1..e9b7da3017 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -236,39 +236,26 @@ rules: ## Sidecar -[//]: # "TODO(kakkoyun): Generate sidecar rules using thanos-mixin." - +[embedmd]:# (../tmp/thanos-sidecar.rules.yaml yaml) ```yaml +name: thanos-sidecar.rules +rules: - alert: ThanosSidecarPrometheusDown - expr: thanos_sidecar_prometheus_up{name="prometheus"} == 0 - for: 5m - labels: - team: TEAM annotations: - summary: Thanos Sidecar cannot connect to Prometheus - impact: Prometheus configuration is not being refreshed - action: Check {{ $labels.kubernetes_pod_name }} pod logs in {{ $labels.kubernetes_namespace}} namespace - dashboard: SIDECAR_URL -- alert: ThanosSidecarBucketOperationsFailed - expr: rate(thanos_objstore_bucket_operation_failures_total{name="prometheus"}[5m]) > 0 + message: Thanos Sidecar {{$labels.job}} {{$labels.pod}} cannot connect to Prometheus. + expr: | + sum by (job, pod) (thanos_sidecar_prometheus_up{job=~"thanos-sidecar.*"} == 0) for: 5m labels: - team: TEAM + severity: critical +- alert: ThanosSidecarUnhealthy annotations: - summary: Thanos Sidecar bucket operations are failing - impact: We will lose metrics data if not fixed in 24h - action: Check {{ $labels.kubernetes_pod_name }} pod logs in {{ $labels.kubernetes_namespace}} namespace - dashboard: SIDECAR_URL -- alert: ThanosSidecarGrpcErrorRate - expr: rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable",name="prometheus"}[5m]) > 0 - for: 5m + message: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for {{ $value + }} seconds. + expr: | + count(time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"}) by (job, pod) >= 300) > 0 labels: - team: TEAM - annotations: - summary: Thanos Sidecar is returning Internal/Unavailable errors - impact: Prometheus queries are failing - action: Check {{ $labels.kubernetes_pod_name }} pod logs in {{ $labels.kubernetes_namespace}} namespace - dashboard: SIDECAR_URL + severity: critical ``` ## Query diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index a1e77d932a..8db6674442 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -212,6 +212,14 @@ groups: severity: warning - name: thanos-sidecar.rules rules: + - alert: ThanosSidecarPrometheusDown + annotations: + message: Thanos Sidecar {{$labels.job}} {{$labels.pod}} cannot connect to Prometheus. + expr: | + sum by (job, pod) (thanos_sidecar_prometheus_up{job=~"thanos-sidecar.*"} == 0) + for: 5m + labels: + severity: critical - alert: ThanosSidecarUnhealthy annotations: message: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for {{ diff --git a/mixin/thanos/alerts/sidecar.libsonnet b/mixin/thanos/alerts/sidecar.libsonnet index 7c80e3af08..58aee3bb0b 100644 --- a/mixin/thanos/alerts/sidecar.libsonnet +++ b/mixin/thanos/alerts/sidecar.libsonnet @@ -9,6 +9,19 @@ { name: 'thanos-sidecar.rules', rules: [ + { + alert: 'ThanosSidecarPrometheusDown', + annotations: { + message: 'Thanos Sidecar {{$labels.job}} {{$labels.pod}} cannot connect to Prometheus.', + }, + expr: ||| + sum by (job, pod) (thanos_sidecar_prometheus_up{%(selector)s} == 0) + ||| % thanos.sidecar, + 'for': '5m', + labels: { + severity: 'critical', + }, + }, { alert: 'ThanosSidecarUnhealthy', annotations: { From 57a1eae62a94f55414f3c47d3bacb88f3c2a4732 Mon Sep 17 00:00:00 2001 From: Kevin Hellemun <17928966+OGKevin@users.noreply.github.com> Date: Wed, 5 Feb 2020 14:38:59 +0100 Subject: [PATCH 223/257] Register thanos_alert_sender_errors_total metric. (#2101) * Register thanos_alert_sender_errors_total metric. https://github.com/thanos-io/thanos/issues/2100 Fixes #2100 Signed-off-by: Kevin Hellemun <17928966+OGKevin@users.noreply.github.com> * Updated CHANGELOG. Signed-off-by: Kevin Hellemun <17928966+OGKevin@users.noreply.github.com> --- CHANGELOG.md | 1 + pkg/alert/alert.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dee286e74c..2f7a8d9066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#2033](https://github.com/thanos-io/thanos/pull/2033) minio-go: Fixed Issue #1494 support Web Identity providers for IAM credentials for AWS EKS - [#1985](https://github.com/thanos-io/thanos/pull/1985) store gateway: Fixed case where series entry is larger than 64KB in index. - [#2051](https://github.com/thanos-io/thanos/pull/2051) ruler: Fixed issue where ruler does not expose shipper metrics. +- [#2101](https://github.com/thanos-io/thanos/pull/2101) ruler: Fixed bug where thanos_alert_sender_errors_total was not registered. ### Added diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go index 7cb11a75bd..418cb5c831 100644 --- a/pkg/alert/alert.go +++ b/pkg/alert/alert.go @@ -313,7 +313,7 @@ func NewSender( }, []string{"alertmanager"}), } if reg != nil { - reg.MustRegister(s.sent, s.dropped, s.latency) + reg.MustRegister(s.sent, s.errs, s.dropped, s.latency) } return s } From 93744bb43610495f248b8dc1c86a81ae3b7d1b58 Mon Sep 17 00:00:00 2001 From: Matthias Loibl Date: Wed, 5 Feb 2020 23:31:08 +0100 Subject: [PATCH 224/257] Add metalmatze as release shepard for v0.11 (#2099) Signed-off-by: Matthias Loibl --- docs/release-process.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-process.md b/docs/release-process.md index 988fc22521..9a490fa4b5 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -32,7 +32,8 @@ Release shepherd responsibilities: | Release | Time of first RC | Shepherd (GitHub handle) | |-----------|--------------------------|--------------------------| -| v0.11.0 | (planned) 2020.02.19 | TBD | +| v0.12.0 | (planned) 2020.04.01 | TBD | +| v0.11.0 | 2020.02.19 | `@metalmatze` | | v0.10.0 | 2020.01.08 | `@GiedriusS` | | v0.9.0 | 2019.11.26 | `@bwplotka` | | v0.8.0 | 2019.10.09 | `@bwplotka` | From 6948d5b375e24970e24705daaab486cca57f0dd9 Mon Sep 17 00:00:00 2001 From: khyatisoneji <31898080+khyatisoneji@users.noreply.github.com> Date: Thu, 6 Feb 2020 04:05:02 +0530 Subject: [PATCH 225/257] fetcher: add filter to remove old blocks if new overlapping blocks found (#2071) Signed-off-by: khyatisoneji --- cmd/thanos/store.go | 1 + pkg/block/fetcher.go | 96 +++++++++++++++++++++ pkg/block/fetcher_test.go | 177 ++++++++++++++++++++++++++++++++++++++ pkg/block/node.go | 53 ++++++++++++ 4 files changed, 327 insertions(+) create mode 100644 pkg/block/node.go diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index b6b7ed320c..ab36a505cc 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -223,6 +223,7 @@ func runStore( metaFetcher, err := block.NewMetaFetcher(logger, fetcherConcurrency, bkt, dataDir, extprom.WrapRegistererWithPrefix("thanos_", reg), block.NewTimePartitionMetaFilter(filterConf.MinTime, filterConf.MaxTime).Filter, block.NewLabelShardedMetaFilter(relabelConfig).Filter, + block.NewDeduplicateFilter().Filter, ) if err != nil { return errors.Wrap(err, "meta fetcher") diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index 6250d01111..b6411e5431 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -11,6 +11,7 @@ import ( "os" "path" "path/filepath" + "sort" "sync" "time" @@ -21,6 +22,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" + "github.com/prometheus/prometheus/tsdb" tsdberrors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/thanos-io/thanos/pkg/block/metadata" @@ -50,6 +52,7 @@ const ( labelExcludedMeta = "label-excluded" timeExcludedMeta = "time-excluded" TooFreshMeta = "too-fresh" + duplicateMeta = "duplicate" ) func newSyncMetrics(r prometheus.Registerer) *syncMetrics { @@ -84,6 +87,7 @@ func newSyncMetrics(r prometheus.Registerer) *syncMetrics { []string{failedMeta}, []string{labelExcludedMeta}, []string{timeExcludedMeta}, + []string{duplicateMeta}, ) if r != nil { r.MustRegister( @@ -410,3 +414,95 @@ func (f *LabelShardedMetaFilter) Filter(metas map[ulid.ULID]*metadata.Meta, sync delete(metas, id) } } + +// DeduplicateFilter is a MetaFetcher filter that filters out older blocks that have exactly the same data. +type DeduplicateFilter struct { + DuplicateIDs []ulid.ULID +} + +// NewDeduplicateFilter creates DeduplicateFilter. +func NewDeduplicateFilter() *DeduplicateFilter { + return &DeduplicateFilter{} +} + +// Filter filters out duplicate blocks that can be formed +// from two or more overlapping blocks that fully submatches the source blocks of the older blocks. +func (f *DeduplicateFilter) Filter(metas map[ulid.ULID]*metadata.Meta, synced GaugeLabeled, _ bool) { + root := NewNode(&metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(uint64(0), nil), + }, + }) + + metaSlice := []*metadata.Meta{} + for _, meta := range metas { + metaSlice = append(metaSlice, meta) + } + sort.Slice(metaSlice, func(i, j int) bool { + ilen := len(metaSlice[i].Compaction.Sources) + jlen := len(metaSlice[j].Compaction.Sources) + + if ilen == jlen { + return metaSlice[i].ULID.Compare(metaSlice[j].ULID) < 0 + } + + return ilen-jlen > 0 + }) + + for _, meta := range metaSlice { + addNodeBySources(root, NewNode(meta)) + } + + duplicateULIDs := getNonRootIDs(root) + for _, id := range duplicateULIDs { + if metas[id] != nil { + f.DuplicateIDs = append(f.DuplicateIDs, id) + } + synced.WithLabelValues(duplicateMeta).Inc() + delete(metas, id) + } +} + +func addNodeBySources(root *Node, add *Node) bool { + var rootNode *Node + for _, node := range root.Children { + parentSources := node.Compaction.Sources + childSources := add.Compaction.Sources + + // Block exists with same sources, add as child. + if contains(parentSources, childSources) && contains(childSources, parentSources) { + node.Children = append(node.Children, add) + return true + } + + // Block's sources are present in other block's sources, add as child. + if contains(parentSources, childSources) { + rootNode = node + break + } + } + + // Block cannot be attached to any child nodes, add it as child of root. + if rootNode == nil { + root.Children = append(root.Children, add) + return true + } + + return addNodeBySources(rootNode, add) +} + +func contains(s1 []ulid.ULID, s2 []ulid.ULID) bool { + for _, a := range s2 { + found := false + for _, e := range s1 { + if a.Compare(e) == 0 { + found = true + break + } + } + if !found { + return false + } + } + return true +} diff --git a/pkg/block/fetcher_test.go b/pkg/block/fetcher_test.go index faf7eafb5a..1f0cfa341d 100644 --- a/pkg/block/fetcher_test.go +++ b/pkg/block/fetcher_test.go @@ -7,10 +7,12 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io/ioutil" "os" "path" "path/filepath" + "runtime" "sort" "testing" "time" @@ -393,3 +395,178 @@ func TestTimePartitionMetaFilter_Filter(t *testing.T) { testutil.Equals(t, expected, input) } + +func TestDeduplicateFilter_Filter(t *testing.T) { + for _, tcase := range []struct { + name string + input map[ulid.ULID][]ulid.ULID + expected []ulid.ULID + }{ + { + name: "3 non compacted blocks in bucket", + input: map[ulid.ULID][]ulid.ULID{ + ULID(1): []ulid.ULID{ULID(1)}, + ULID(2): []ulid.ULID{ULID(2)}, + ULID(3): []ulid.ULID{ULID(3)}, + }, + expected: []ulid.ULID{ + ULID(1), + ULID(2), + ULID(3), + }, + }, + { + name: "compacted block with sources in bucket", + input: map[ulid.ULID][]ulid.ULID{ + ULID(6): []ulid.ULID{ULID(6)}, + ULID(4): []ulid.ULID{ULID(1), ULID(3), ULID(2)}, + ULID(5): []ulid.ULID{ULID(5)}, + }, + expected: []ulid.ULID{ + ULID(4), + ULID(5), + ULID(6), + }, + }, + { + name: "two compacted blocks with same sources", + input: map[ulid.ULID][]ulid.ULID{ + ULID(5): []ulid.ULID{ULID(5)}, + ULID(6): []ulid.ULID{ULID(6)}, + ULID(3): []ulid.ULID{ULID(1), ULID(2)}, + ULID(4): []ulid.ULID{ULID(1), ULID(2)}, + }, + expected: []ulid.ULID{ + ULID(3), + ULID(5), + ULID(6), + }, + }, + { + name: "two compacted blocks with overlapping sources", + input: map[ulid.ULID][]ulid.ULID{ + ULID(4): []ulid.ULID{ULID(1), ULID(2)}, + ULID(6): []ulid.ULID{ULID(6)}, + ULID(5): []ulid.ULID{ULID(1), ULID(3), ULID(2)}, + }, + expected: []ulid.ULID{ + ULID(5), + ULID(6), + }, + }, + { + name: "3 non compacted blocks and compacted block of level 2 in bucket", + input: map[ulid.ULID][]ulid.ULID{ + ULID(6): []ulid.ULID{ULID(6)}, + ULID(1): []ulid.ULID{ULID(1)}, + ULID(2): []ulid.ULID{ULID(2)}, + ULID(3): []ulid.ULID{ULID(3)}, + ULID(4): []ulid.ULID{ULID(2), ULID(1), ULID(3)}, + }, + expected: []ulid.ULID{ + ULID(4), + ULID(6), + }, + }, + { + name: "3 compacted blocks of level 2 and one compacted block of level 3 in bucket", + input: map[ulid.ULID][]ulid.ULID{ + ULID(10): []ulid.ULID{ULID(1), ULID(2), ULID(3)}, + ULID(11): []ulid.ULID{ULID(6), ULID(4), ULID(5)}, + ULID(14): []ulid.ULID{ULID(14)}, + ULID(1): []ulid.ULID{ULID(1)}, + ULID(13): []ulid.ULID{ULID(1), ULID(6), ULID(2), ULID(3), ULID(5), ULID(7), ULID(4), ULID(8), ULID(9)}, + ULID(12): []ulid.ULID{ULID(7), ULID(9), ULID(8)}, + }, + expected: []ulid.ULID{ + ULID(14), + ULID(13), + }, + }, + { + name: "compacted blocks with overlapping sources", + input: map[ulid.ULID][]ulid.ULID{ + ULID(8): []ulid.ULID{ULID(1), ULID(3), ULID(2), ULID(4)}, + ULID(1): []ulid.ULID{ULID(1)}, + ULID(5): []ulid.ULID{ULID(1), ULID(2)}, + ULID(6): []ulid.ULID{ULID(1), ULID(3), ULID(2), ULID(4)}, + ULID(7): []ulid.ULID{ULID(3), ULID(1), ULID(2)}, + }, + expected: []ulid.ULID{ + ULID(6), + }, + }, + { + name: "compacted blocks of level 3 with overlapping sources of equal length", + input: map[ulid.ULID][]ulid.ULID{ + ULID(10): []ulid.ULID{ULID(1), ULID(2), ULID(6), ULID(7)}, + ULID(1): []ulid.ULID{ULID(1)}, + ULID(11): []ulid.ULID{ULID(6), ULID(8), ULID(1), ULID(2)}, + }, + expected: []ulid.ULID{ + ULID(10), + ULID(11), + }, + }, + { + name: "compacted blocks of level 3 with overlapping sources of different length", + input: map[ulid.ULID][]ulid.ULID{ + ULID(10): []ulid.ULID{ULID(6), ULID(7), ULID(1), ULID(2)}, + ULID(1): []ulid.ULID{ULID(1)}, + ULID(5): []ulid.ULID{ULID(1), ULID(2)}, + ULID(11): []ulid.ULID{ULID(2), ULID(3), ULID(1)}, + }, + expected: []ulid.ULID{ + ULID(10), + ULID(11), + }, + }, + } { + f := NewDeduplicateFilter() + if ok := t.Run(tcase.name, func(t *testing.T) { + synced := prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"state"}) + metas := make(map[ulid.ULID]*metadata.Meta) + inputLen := len(tcase.input) + for id, sources := range tcase.input { + metas[id] = &metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ + ULID: id, + Compaction: tsdb.BlockMetaCompaction{ + Sources: sources, + }, + }, + } + } + f.Filter(metas, synced, false) + + compareSliceWithMapKeys(t, metas, tcase.expected) + testutil.Equals(t, float64(inputLen-len(tcase.expected)), promtest.ToFloat64(synced.WithLabelValues(duplicateMeta))) + }); !ok { + return + } + } +} + +func compareSliceWithMapKeys(tb testing.TB, m map[ulid.ULID]*metadata.Meta, s []ulid.ULID) { + _, file, line, _ := runtime.Caller(1) + matching := true + if len(m) != len(s) { + matching = false + } + + for _, val := range s { + if m[val] == nil { + matching = false + break + } + } + + if !matching { + var mapKeys []ulid.ULID + for id := range m { + mapKeys = append(mapKeys, id) + } + fmt.Printf("\033[31m%s:%d:\n\n\texp keys: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, mapKeys, s) + tb.FailNow() + } +} diff --git a/pkg/block/node.go b/pkg/block/node.go new file mode 100644 index 0000000000..1423e31ed5 --- /dev/null +++ b/pkg/block/node.go @@ -0,0 +1,53 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package block + +import ( + "github.com/oklog/ulid" + "github.com/thanos-io/thanos/pkg/block/metadata" +) + +// Node type represents a node of a tree. +type Node struct { + metadata.Meta + Children []*Node +} + +// NewNode creates a new node with children as empty slice. +func NewNode(meta *metadata.Meta) *Node { + return &Node{ + Meta: *meta, + Children: []*Node{}, + } +} + +// getNonRootIDs returns list of ids which are not on root level. +func getNonRootIDs(root *Node) []ulid.ULID { + var ulids []ulid.ULID + for _, node := range root.Children { + ulids = append(ulids, childrenToULIDs(node)...) + ulids = remove(ulids, node.ULID) + } + return ulids +} + +func childrenToULIDs(a *Node) []ulid.ULID { + var ulids = []ulid.ULID{a.ULID} + for _, childNode := range a.Children { + ulids = append(ulids, childrenToULIDs(childNode)...) + } + return ulids +} + +func remove(items []ulid.ULID, item ulid.ULID) []ulid.ULID { + newitems := []ulid.ULID{} + + for _, i := range items { + if i.Compare(item) != 0 { + newitems = append(newitems, i) + } + } + + return newitems +} From 1b22a17224f7c6b92ff46885965653180dab9050 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 6 Feb 2020 19:36:02 +0800 Subject: [PATCH 226/257] comment all log format (#2000) Signed-off-by: Xiang Dai <764524258@qq.com> Signed-off-by: Xiang Dai --- cmd/thanos/main.go | 2 +- docs/components/bucket.md | 12 +++++++----- docs/components/check.md | 4 ++-- docs/components/compact.md | 3 ++- docs/components/query.md | 3 ++- docs/components/rule.md | 3 ++- docs/components/sidecar.md | 3 ++- docs/components/store.md | 3 ++- 8 files changed, 20 insertions(+), 13 deletions(-) diff --git a/cmd/thanos/main.go b/cmd/thanos/main.go index c086782a7d..9c38acca4b 100644 --- a/cmd/thanos/main.go +++ b/cmd/thanos/main.go @@ -49,7 +49,7 @@ func main() { logLevel := app.Flag("log.level", "Log filtering level."). Default("info").Enum("error", "warn", "info", "debug") - logFormat := app.Flag("log.format", "Log format to use."). + logFormat := app.Flag("log.format", "Log format to use. Possible options: logfmt or json."). Default(logFormatLogfmt).Enum(logFormatLogfmt, logFormatJson) tracingConfig := regCommonTracingFlags(app) diff --git a/docs/components/bucket.md b/docs/components/bucket.md index 6be9d81848..20efeeea47 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -41,7 +41,7 @@ Flags: --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: @@ -102,7 +102,8 @@ Flags: --help-long and --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or + json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: @@ -173,7 +174,7 @@ Flags: --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: @@ -238,7 +239,7 @@ Flags: --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: @@ -283,7 +284,8 @@ Flags: and --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or + json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: diff --git a/docs/components/check.md b/docs/components/check.md index d654c5099a..8c0599a7f4 100644 --- a/docs/components/check.md +++ b/docs/components/check.md @@ -22,7 +22,7 @@ Flags: --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: @@ -67,7 +67,7 @@ Flags: --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: diff --git a/docs/components/compact.md b/docs/components/compact.md index c41b1ff2bc..edd71eb7b1 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -77,7 +77,8 @@ Flags: and --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or + json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: diff --git a/docs/components/query.md b/docs/components/query.md index 81c0265f58..96f12e3217 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -249,7 +249,8 @@ Flags: --help-long and --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or + json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: diff --git a/docs/components/rule.md b/docs/components/rule.md index 3f7cd556fc..ffa3dfa6fc 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -240,7 +240,8 @@ Flags: --help-long and --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or + json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 6fd960343f..723dab0775 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -92,7 +92,8 @@ Flags: --help-long and --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or + json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: diff --git a/docs/components/store.md b/docs/components/store.md index 2cc130d9e9..9bc10648ea 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -39,7 +39,8 @@ Flags: --help-long and --help-man). --version Show application version. --log.level=info Log filtering level. - --log.format=logfmt Log format to use. + --log.format=logfmt Log format to use. Possible options: logfmt or + json. --tracing.config-file= Path to YAML file with tracing configuration. See format details: From a79edc7624391487348e64a5292b36094dcbd0ca Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 6 Feb 2020 19:44:38 +0800 Subject: [PATCH 227/257] Store: Document --chunk-pool-size better (#2057) Signed-off-by: Xiang Dai <764524258@qq.com> --- cmd/thanos/store.go | 2 +- docs/components/store.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index ab36a505cc..bb2a547d68 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -50,7 +50,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { "YAML file that contains index cache configuration. See format details: https://thanos.io/components/store.md/#index-cache", false) - chunkPoolSize := cmd.Flag("chunk-pool-size", "Maximum size of concurrently allocatable bytes for chunks."). + chunkPoolSize := cmd.Flag("chunk-pool-size", "Maximum size of concurrently allocatable bytes reserved strictly to reuse for chunks in memory."). Default("2GB").Bytes() maxSampleCount := cmd.Flag("store.grpc.series-sample-limit", diff --git a/docs/components/store.md b/docs/components/store.md index 9bc10648ea..3bda0a7351 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -83,7 +83,8 @@ Flags: details: https://thanos.io/components/store.md/#index-cache --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes - for chunks. + reserved strictly to reuse for chunks in + memory. --store.grpc.series-sample-limit=0 Maximum amount of samples returned via a single Series call. 0 means no limit. NOTE: For From cf558de26c2e242da7cd198e0faa22d324a5a429 Mon Sep 17 00:00:00 2001 From: Aleksey Sin Date: Thu, 6 Feb 2020 15:11:02 +0300 Subject: [PATCH 228/257] Support downsampling for API method /series. (#2003) Signed-off-by: Aleskey Sin --- pkg/query/api/v1.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/query/api/v1.go b/pkg/query/api/v1.go index 74d0effc8c..cc6a20a0a7 100644 --- a/pkg/query/api/v1.go +++ b/pkg/query/api/v1.go @@ -505,8 +505,7 @@ func (api *API) series(r *http.Request) (interface{}, []error, *ApiError) { return nil, nil, apiErr } - // TODO(bwplotka): Support downsampling? - q, err := api.queryableCreate(enableDedup, replicaLabels, 0, enablePartialResponse, true). + q, err := api.queryableCreate(enableDedup, replicaLabels, math.MaxInt64, enablePartialResponse, true). Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end)) if err != nil { return nil, nil, &ApiError{errorExec, err} From d500d7014d73c409eb7d2db81f389f407097c949 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 6 Feb 2020 19:58:54 -0500 Subject: [PATCH 229/257] fix Banzai Cloud blog post (#2106) Signed-off-by: yeya24 --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index df31cedea0..45a375a51a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -126,7 +126,7 @@ We also have example Grafana dashboards [here](/examples/dashboards/dashboards.m * 2020: - * [Banzai Cloud user story](https://monzo.com/blog/2018/07/27/how-we-monitor-monzo) + * [Banzai Cloud user story](https://banzaicloud.com/blog/multi-cluster-monitoring/) ## Integrations From f034581bffbb15324a7608236282e035f2c989d2 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 7 Feb 2020 15:22:59 +0800 Subject: [PATCH 230/257] pkg/store/proxy.go: optimize help info (#2096) Signed-off-by: Xiang Dai <764524258@qq.com> --- pkg/store/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 9d2c677548..1af4242d28 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -294,7 +294,7 @@ func (s *ProxyStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSe level.Debug(s.logger).Log("msg", strings.Join(storeDebugMsgs, ";")) if len(seriesSet) == 0 { // This is indicates that configured StoreAPIs are not the ones end user expects. - err := errors.New("No store matched for this query") + err := errors.New("No StoreAPIs matched for this query") level.Warn(s.logger).Log("err", err, "stores", strings.Join(storeDebugMsgs, ";")) respSender.send(storepb.NewWarnSeriesResponse(err)) return nil From 5a93f510c932425f69c387c0767db3ea504a55b8 Mon Sep 17 00:00:00 2001 From: Anas Date: Sat, 8 Feb 2020 13:36:08 +0100 Subject: [PATCH 231/257] Avoid breaking the json log with series string (#2104) Signed-off-by: Anas --- pkg/compact/downsample/streamed_block_writer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/compact/downsample/streamed_block_writer.go b/pkg/compact/downsample/streamed_block_writer.go index 1cfad55749..267a93bfab 100644 --- a/pkg/compact/downsample/streamed_block_writer.go +++ b/pkg/compact/downsample/streamed_block_writer.go @@ -7,6 +7,7 @@ import ( "context" "io" "path/filepath" + "strings" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -111,7 +112,7 @@ func (w *streamedBlockWriter) WriteSeries(lset labels.Labels, chunks []chunks.Me } if len(chunks) == 0 { - level.Warn(w.logger).Log("empty chunks happened, skip series", lset) + level.Warn(w.logger).Log("msg", "empty chunks happened, skip series", "series", strings.ReplaceAll(lset.String(), "\"", "'")) return nil } From 800d1e82e3a97bfe72b529f4194251fdb2becf97 Mon Sep 17 00:00:00 2001 From: JacktheLee Date: Sun, 9 Feb 2020 03:58:59 +0900 Subject: [PATCH 232/257] adding Toss as adopter (#2109) Signed-off-by: JacksChoco --- website/data/adopters.yml | 3 +++ website/static/logos/toss.png | Bin 0 -> 10611 bytes 2 files changed, 3 insertions(+) create mode 100644 website/static/logos/toss.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index e7a700e22e..c7355630c5 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -84,3 +84,6 @@ adopters: - name: Tangent Solutions url: https://www.tangentsolutions.co.za/ logo: tangentsolutions.png +- name: Toss + url: https://toss.im/en + logo: toss.png \ No newline at end of file diff --git a/website/static/logos/toss.png b/website/static/logos/toss.png new file mode 100644 index 0000000000000000000000000000000000000000..67c1d98e8bad0b089fa167d633a8a365b0f0e95c GIT binary patch literal 10611 zcmeHs_g7O}(Dni82q;MJB26KPQ~?QIq$%A(F@VyMW+(;_2v(#>RYD6vs*3bp!xaJ| zReBMS(0d6GN=Uwg@A}^V;a%&VUrw^l-g{=AnLW?U-Vvt8dMr$QOaK6|80gb}pkbo8y z0H`Gn)IP{l9)XY|gUalF*}$=+DDBpK?RRD9S-xxqHj$j|a)hcA*5nZ5xX{;Pym zIit1V3O#PMI4;kbt)=L0MSJ!3VpaeS-%OFUQ^38Lf>~@IMbsasZLk3)75R)|sflHe z<9}mi$sSLLd;VsBR=-MJolIjk&Mzug9T(O+p%*8O;P>F4&8PrcAZ(GNRCPxU!cIPL zDH1D%>o!+NF&K8Ab6wx#;vm=bAn)Ok_d)ed0U$tvF{*=n+&4uSa*+QMWv}I4RJ-R# z8DJu>f?$AnHJccWf{ltP`20icDzSEVHdx)r?A2c4`ZDkrgN_Leu07%35w#BB#W_Rfouip)mPYxEC~)t zI`D=uB)hVnIx9?BoP1nhww^q=kN8SaeiPO?WEfGQEBN=@w?vAWKL0oLljy>n{X4Xv zYeDPF?vRL^MnqxPI5ZkEK}Z_|%>m_H7pG)DszY+x?+Bx41l&Ob!Nv0uDY2sOk0x+U ze5E$fB4-O#X>c?xkof{sBsELu<*(&bYmJ;SAPJ=POaK^6BDFgzN>CDh1r?%bET2%F zYHb?0gRCO)svIi_Hu4@9>>!Y+6K-(657wP}$p|m3ho03% z=8+cX131%`r{eR#Pq&=`EocJe#T?W<(cbxgka;LckzcMJ!(9C953Tk@it3JcIaCMy z3tHL%P2r^SILmOVfym+J3kN+K>Qfc<$;YBX=T(Q#0>B5He3;0g%!xDq=Jt4NRzNP+gviBb4g-yTR4qqI-bE&n5I;lBXjEa(y1W$WWs$l# z>iH6VWAf#q&0kl@CwtTQ#;bBYc22GtkD{YB*XjI5L){T773F z`LeiKttZFKfhA`u5zhyQh=txy{_=kkUyFZpaS4`6=&G{(hUfl%-vxCmf*F+B&}XG; zOZIaPMz&Ta(=&m-K{0iU2~lb`M^YFaf8(JPF~&fao5H}Zr>jD@=fi?I}NDu zrs&PDci~jDfJixOi_~B>U21S^&rk|=7`4``2nsy@TSZ9AFC944AA2sYNe=Qb;{f{c zX-37fx$C%jhovd<4JIm3{D4hI(>A>Fct9-<9o66H$>6i`G`;;LNeO z1O;P#pT`Iu5kawph1H$pF=qM;stZqo5C#`-%e<+hHjv9r{2`y zUUpaaov_#wEk+g&_8-D;**?Ty+c1FD`P;n!ccbafVhuKwpVNT%UdrdQksP`ZZ75Sr z>A9zkq%KU};D^^hW5FSm<6Li!=!^cFZgn7GC-BG?`vduSIije?e0vd2xeRL)V5aJP1I7SO}j z`cQ5)9vj-FDHlG(e~$jgdKKEwLs4LKJKDKQ7{w9YypzNCqehMRVxm}{Bj0zbbdG{q zKcI;N&J``R6`;3rPr=kY%SYBMaIj?i*3SH>72p1@!cJNIXf7`=ey8G&&S{bn+PoEg zDNw_~o(oXdbbyFo=9f%pPr-RfJ>!)~!)GyoK59-vzY9S#C}(Q5LOQUl2Md~9o=Mtk zbfVqkBKeL42*i0n3XB7wZ^d zstVPXvLB%k%ConCDg*Pj)K9rOVSagaA&;v}l=V1q^MjCsp?ZmJ8Wpv^&JH=)dVWP;k6Y)Ow~l8NH2FFIkd zN|c`1)DV|dBS9igyuK#5Du?9)km)VBn|#2p^N8P>YU_9TyycEv#O(O6T>iAf>k5JS z^+62&T7OP1&+Cd?jHW=?)YUyqVg08Vnz;xXaS2IXse)jy5e0uMcn!&Ap|r-)o8KYY=RPfLO+1ld`3lx;|l41|D35>+70iJ>DjaYjBD z)E9M^M(@Muo_^x3Sv23Vi#oY@UgO)=lvn@I&Ys)sYmd5W2y53yNH0s|Yl60}!D7W3 z_+gpH<}xchVq~M{Kqfn;l{VldFA!%!^+zUISN}R*YVKl-`|q=1-5*rWH2dZ(ZLcm-+A5nagRB>`Mg3^dpPjM_V#}0kieT7%W%`%0y zT)e09mGl7itZoxZn|)%ct^j1Zefb$$C@NJ;s6x$DhCc9Ve|~R3J+@=nppbF9D7(G; zy`fj|o3}My*FDXZk@GJ67fNGCnpJ0CY6U95S1>p3lCQn()?B>w!J_7C=)J(;k$`R-{t=Igb3g^MGU5G*yTqnusX3 zY?>-`HW`%X7bnLk-kmKun7pnEvG32jMR1=jOk!_CF-gkv#@vyax-eonX|PZd4v*-R zj=2_l${AzN1|?bi)|O)9C6;nk@#!r~$hBQz#j*m#Wts-YLyy;ERSrDfWdYo7l-KP= z7%P4LnUhj$wrPy?e3T*H=l<_$JT+KDw5z|VFtRQ;K;i{CDDaj|e4aja3?I5Eo&BTg z%i!dj|EQ0_xwJ4Y1Od+NPWZh;=#7{=4b@!#xxHA=BZen+ELlqrFn0DxC*{j%ukIYk@G<_dD9^DO5oB z)gu!@y3|aIsXJt0qX!G3-88N{jYg5Ye4bkf=j}wt$Wwb|op&)uqYNd`-jDECy&r=U zDfN!?c{3NAA)N{Yui7VV93lpMlef#e*X)30g$59_RlRKB@6Y*n~^|NoHMoJ)f#M#e@CBdUuQR_PUi zfHfTx5C^|FJjZ8q)TuNQV&uWfmDo=huCi2f2pw+r4#he>R5@Db?Dq&;9gC=D?b76Go{)^)AZIPjJ9e#L)e%Oj zTP)ZM5@We3#z=32tQz!8GBdBcLtV`P=haqVoN%A7>iu4$uQuBirpj~zSL!T8X7?$; zsm2!&MnEM9{#`#(U10=ub71(mTxgO(H_-rA2Cvk~N^zI>wrq&G%M`>P#kswwlwR^k}JbQ9N5UN-rYu zBr2(DGZc<)+x)&7H~h7Ic^TC?0xv+6nZQ#JQV^)xdG0?}U|IVy^#+E2X`hh0!Fpt$ zvSp|~5(AGsI(WvlBAa)%T7Su5d)`6MI^+glV^h*>FVCyJvKLlSns=f*o%Hb67>F({ z73z>?jxHeV2~N)3z2wDLwbjbJpMqYUNIv~AyVWO$D?1m~%`9VeW4$NJvrqiw+MIrs z*9fS)ow8>87>VtNbi{8QinV(F+rl()ge$_?kPO|M8@KJvsK;i9^cXA#_TqO+#s5Z+ z?s5RcE>cU=U=C8JaiX$*&u-Cr<3x6P(aLo*m0)ugd0S1=O3)MKO!e-L*WVe=x|A2B zgdgi&L46tip>4ldDH~?J#mC8hMJ{wOXT>8KNMPG(4DhKQv3<5h3$AHKdxd)A>J8O< zF*PI$+7bh0#{8N&UT!efpIrMzqZQ5sQ1&x#a95J6Y}m6wUer&c_93Dtg1<}u|D_s6 zXdLmn9HuX8doAlG*|WeV!jbUnwL*~&)V+j31Y*KX-J?hgQVnsG#1_F{VKVN7_1v^7 zJ1kJ8J*gP|;3WmIa#MzY#mBwYC1{@#lR+Ri6-2RB&ORZqaXh7|wkDmqB^z$gP7h|F zzM$oUw1~GNV$4Qbj2+Rrmjc@*y)=ms%N;`b0Xn$W`qh*ntfzI}K$t&@8Emwik^n`j z6mdFmX6*BjI#S@dBe>(NyAhuZb!$yS1d`*e-t0~-kH+a{{b)#$l+5$I%c>qwoy$;S z*~F^N=Wpv8=Mf30bdIRQ14+we!6CzE!6^1z7l~D*;gw(iP$mUb6r3x`V0pT6Jf1uos)*Ev2n_qK+rhv}_r^i=E8{85mvc|eqBYv3mxiHl@<=)L5h?nI%g;j_82KFc zF%^h&MGj3S@KRKL0pb{-Roc(%L_$C=&Kf?(NG;naAysuBOl&Y{K{ zfqf~6umB6g(4h@Sa|{Tri88R(wE1ajOi%vi#)7M(HA3U_w#q{NnP9b>kE|;?L$^=3 z94N|u-g55_4kdK#(bdaSGQZ^n-h1m2@z-QD@1XGPE-vYTT_BS?j`$v&rI(kje4yI`T#So$#K<$A`=2;z!%)oBHc_3=X~AkaEFGI8(%vHq z!gWY|k1}X8h`;cxJ3JL>_dFk-->4(hM0qBOc{OzGeVidZ8Qocv9DiCTNq`m%6lF>* zX@5WolD9q4SU5C?*Th2bc}(m-Cwf4SIzm=6BRD2f{<;JlZC4G$;~a+h28k<>2E23BN! z4mH}Yww)aAbDb7=MtHyx=fG{kv8y|Y%TSd?^rsBp=`fjm^VRWePZ6J`i z_~|h5yv$;KROhc9Z=@qT_g|Piq7|ulkMuZpzm`wkq7sSuDOlaKj;C|z)@bkU3k+as zKfhEyKh4VL#ZMa+4zHEQ@8r$YEV+ZNn&Ygukygy4vXRJwlB(z4;BUoOs}5(dfsfzz zl)nj670djhG=2PX%TYA5qlTn+4ph=dzMsYs#c&&Pl*|+C_Sp#XP+%aL#Ku>v7brc` zz~HtLIJve=@PVaDQ75NRfjs%bFSfp@|QH%-bVI2 zU5NzT7KRt;rvJ@`^ek6wL_xOi3S;7`0k z!M*uo!pu5hF5o#z-T0hgOaUKs^HHV7tXBn+q;5Uq4d#?W*=^+T@XSX2^mHBL?1hLj<%;z6Lg7^0kBEhFrByX6ek{3 z9ZFl;OS=Q4#*u=c6C|k&YTZP(L^tvyCL1w+hD7IoWq=LXp3iM(1fX)V0ASb??kcgTts$~y}-_npE3HvjGM=v3R z#F}8D5*wJ(D=w>5miqGOy_=FzkEHJ$xig^C`{qTQu5QnB^5LqHN1EXimBh}yt1=0w zPHT?j4`FCo%D(JeIea!S0^?otWZ(nLHtB$u-by@eImGY@xsbpFT(dA; zy>)_}l$M}bw&;rNz-gQDgmzj(n z7?_P8c$jMAv9=EAr~-8UQ>ef?g{ul`Yi@jafPG{9l8}KVb zBFrM>5NF)cE=akeA}UBYjllLVa(SG=4uyBJcw=S$uwH{uJZ5>T$%Z%Em!N}FFi z5hJ)8!@|r*Y0lZYp!O)G96WjpKgUXMJfm)_h7S}wJYLS_P^X|uoDu4)=W8v0)Q?KB zN^i&VzkI)OV&|~)v@X&q1%J#t-R9EWe%3*m(M?*%qufeAExjko$|y}jv;}XK?%t1m zMqZhtc&$8dctFR}xOs9vi@W%fszM?85~1!zuLdVdX7(Dh*79krgfNmv@tsIBKbqpp z0nqEQyJk-Lu62l|P}uuc8x;~nNTQ65CGwX4?swnW%mlMWMWw*etap&_1FIqQEE2X8 zp-HbSxR?7CH-&jaVghV3JYbn)>p|}s_uJn#^~PkJ#PrBPqVP9H&ZCaYlXm|02AoL7 zrm8chHh7AA{<3*k(7Z@X_NUYuNr&)Vtl+_=p$mR898daZhC8f$?>x-F>JDUx4KNiA z+>}DhHjIpWNE<_E*_p4+(m|(%o$J7Sri*d;RC^o0K_wB*Wt%5HDNW~Zm~ z{M?d_4*DRVFul(72`B1J`r5hjfE(@Kjbaq)@x+G>#@IE*qx49`w)$QO-LwrHnvu;i zO^6k0b-J*zulfV}w<=0@Z5WJil1?a(qMLfyBdGgR!A_8)ZdU85zD55r`dOv7 zv{wD*FU2EdxkH!jQBw!*+d&uQZN2!_DEjBZzXj3P8Q4&`IeP-{eY|;oi*=GJm{zvB zxGBEbYuq|J$bmn{Q3i3jz~s;xYv17%Dy^r>?bQ=!gw(kzg3x-;T2ceY{0IbVX4J40(o0HbvhB>>)*lhn%kstPExr$0EhbPT)gmXiKP8km4SdFYsTfk1J{00CSIa2C z_kXMb+Z)rYL1`>Bh%4~~QNXlU<)UFRM0MgX=`-#<{-ZzPK}FZsE#>te`BuWrmOAQ2 z+GFi3KGQnO$4xoQy&K0XT~(>}8Ic<7gTL{ogxo>LrWWKj2*u`ziEG3Cz8BR_%CqUF z+wPczYj(~}jsGsgux*d|&-+ME4jHGBZwM>sOs)v5pF=6YzeN8d4z9puA0B+Z|EMSN zTWqdDHb;|Iy(p)jz^mPUB`C|}<)DynC586=3jV)7INP=u&u_lW@*3;8liUcl6CYc3 z9)kz`BC!|5a?mebWDVXGy_Al$*)n!>vq&X^`8n78gi}fEZecn~VO*RetYGyy@m#+=DJGK@(XuIvmQJ=Dz zKpMHO-fkf0G-*MfyD=$tT66~_@B)2KZco0(H|^KPk+4VBv4TG!-PE7Gw43YqBePLS zXBwuy?ax7RGMXG~^kr#?T*(x?`CG9L=wSOw`!A0d#>?IgM&w& zY5j(YSWo?>6bpQRc@2QQdIqT5@XF~m1Uhtti}s2tr=3{VxhA?YMCH^w@MJ}Nx=VsI zBmwEl2%g@^bO+~0pznK4#D9vs^tUaDk~Y)wfc8LmY6pA?lm>cYd`zf0rMXm z>tZia;;#SitS=5JG3#rQL3y0{G{j~F{EI;Sw8Z&pi+^rsu&O^RtcG?NjzrezlW$T1 zKfW?t9pD*LMUgCg(1_)=-J3kq8AF55qUKjcEZIw%R0vZk8iE{rvrY=_;luSm?INOT z05&UXAjE05Sb&H2%RXOhrwzUg1u?`G?prqJPsnLX9#wg`a`gvM0y~R{vvI~TOUHu* z9xS2~)4$~h!xO*d^3~!%XNP4-=(~LSncWruNqKJ3S?4tpU;#4!zA=Y^Taa<^(oJgL zyO+oovQ1O;I%U^4W+sJZxm`SAjy$M4oKSI!iRaY^pfn(qv!-1)W+_>^{DG5o6 zpV~G>$u_&MFn)3D@>|boZE>9Vl%9E`!;M_7U)E}dq#KK=ZrVt%fV z%fH+7#_SdoFN!`X8@Rzd$ip*WBye@HU_W)@fMDT5RJ5%&Dk#KJ}6lZ!`Pi6ia z^KJ~7ko1$(J@K!zThc(vs`W;(RxJiOH&0~U2v?eVPPjj@ADBbg+vdA45<0mZ)Rpya z7Bz$kF@_MLJ1q>LV23!l5!MgT>q{cu8oXI=veR$l)o`H23_Cw{OJ2+b7lgT+|+b4BOx?o>!$rpaUzAh+w0(iEI)F|+c^XL_xy*ze+c}C!2e+c ew#q2nj{s^B>F99QpFXFC80Z+^D*ETq%l`*|&ahqp literal 0 HcmV?d00001 From ecb4018c897754d9c26bb683533f955411591a6c Mon Sep 17 00:00:00 2001 From: Dmitry Ulyanov Date: Sun, 9 Feb 2020 00:18:34 +0300 Subject: [PATCH 233/257] Add Ozon to Thanos adopters list (#2110) Signed-off-by: Dmitry Ulyanov --- website/data/adopters.yml | 5 ++++- website/static/logos/ozon.png | Bin 0 -> 13025 bytes 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 website/static/logos/ozon.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index c7355630c5..c0a498564a 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -86,4 +86,7 @@ adopters: logo: tangentsolutions.png - name: Toss url: https://toss.im/en - logo: toss.png \ No newline at end of file + logo: toss.png +- name: Ozon + url: https://www.ozon.ru + logo: ozon.png diff --git a/website/static/logos/ozon.png b/website/static/logos/ozon.png new file mode 100644 index 0000000000000000000000000000000000000000..14608b957a48755d713d3d014c5546db482720f1 GIT binary patch literal 13025 zcmeHt_dnb1_rE===t66^iq@#M2vVbNHQQ8CV(-02gCOSJMNwPrs!|e3(Ab13YDSBw ztwus(@9q2Q{rwj{Kfd!r@_6KRjdPvXIp_I2=Ui`}J=JDE$9s;3hK5~N=aC@|4FiUT zhE9Zq3HYbISP=pITzjEu`ohS^`9+YOzY~qRqmR9lu&$S#i<6;~og>_@%SoAr=6;Os zBlYLOW6P71#+FLNySDeJ4MI~a_5A8{f4|y3W9r`&Ko}k}{J!|CdG-=N?E=da?HLM7 zhd>nm5_mW={@$|)Z3{w`$*61-sYLIWQu6NTn7Amg`~N=w)4=~U@PDC!0%?mZn&9LS z8|ErC=EQw{>(5o?GQ8&_*?ZN4a=jpOf;2;q*U!~9-M-p?I+2G6bk08ujBsfaJ;o3e zIG|MowaIB4&s#KExdDOoN-poc=?w9KXHtkQ0`$-Bv1~5lv{_ysQFrR%Vqqk}NRurd z%;@EPb+f}ZQ_q)DBky@3+&cGdEM=(lJWt_IiFvL)lM1n#iyo5ilYx65$WJnq)p{g* z3$|D)h5ud-BR6+Abo2~ZvZw(9SBT*!NAvLn)q}Mk`i)y+Gxoc1nfJrgJ}<%7_|Jj& z@0#}G5Xuec&aA>9KBQfJ*VF>sAylw8E3@I7wWM~bvLb&i#*eW7`_$QM``OGiE5rV) z8LlC9(EN&)ID<%k`_t&zn@hD}&E9Y?` z&1(D&`U5vwCBOK@Y-qioF^FpmDs^;#f2%uuISNm9Yjx8!!)iVW9}gNX_80p9}+4Z!tMZXHtsTOK`g+ z*-=%P_EhlURfL0QrIwI%QP`+0(Je6^X4bMJj z**8*?G5MCA7`Bb8TlDx{I<4Fg zEXC(hJ~CbS0a~-rJU;SeI6kT|JYq1$$6uaLO$Nqdz-rw}5^q$O(8Rpvb#Q2`SRl{I zsSO{cGC<9Qw3#Bptr85o#9&sNs293a zTQwv-F(#z-AA!SN;(|se^IJU7Tl3HWIp zo{RA_Tqu{z$<*uT<+RdXm9Bg%#b2XCby;uWNShto#TOb1xTslkcQA>?UCT=iD`?D= zYpnI}m6KqD?R;AXj4h5Y>2mh_yKEi$@kG{ZlyG9f3K**7`Ycv8bfC5PT|InN$7p*> zm&fyK^~8iqXh|}z>9WwjH-Z_FABIRf<;nta3t!oEs$~U6`!GzgEHoSB`v_70wbNyU;!?iB9w*=71pZNNhO`5dIHfWz=>s(}Z zucbD_Qp4nPch&A2YZ$lyN(542@}OB9l|REJ<)cwK=%vLVH)dA*h|~6356UnUhVDiV ztBHT88VQ4)8UR?PbF=j2*uJxsI`1=hVBbp=@ z>((s=Q99@Z_r1s+&6-L_uc11LPKK@H8A$Xac^%Qn&zYeSr7?_u-vVJglf}|6-;@?b zjCHGW2S(PzsmsqY1|u)iCR?Y7GXHx$m{ET^bDq)sPNT=i4n$!6#>MtW8I=sc(2x^Y zsvbgei(cB`R^v32)!92a#%HpvlEGsgjt|n_NwUcaTK~vUh|^p*2&yaqQ7qFuPM-%2 zeu0xVo*8W$!q*QEsc(yE+(m&c0+Jcf!8r!+&4qdDlNncDt44q=Iob^ZI!+3N(gMe6lvP;MVJ`F$m0)^394J-DuC223~R=Lgjg(FH&=cvmloyg37|d}(;Uv6xzW{D8NBsH zd8GLd*ys8EN2thZTRqFNi>Ek7!U<^(H6>yq8i^ZmN)i~GQ1JRStPuwmx;lQ66-wH~ zpOG(~P1>w2X#@144|MP17)cix}WoR&Wv3f%B8EzIt+;>M1?L z>6mE3-D?KDZUiPPD5m!r-H8pUmPa2#=i8U57@kS7=~vjkf^XKfHpX<&j8$U~zclY? zqqB*_Jsan}45a+_&T%tt&})dyX-hiG8}6b-B@Po}yne{!ba~Xqyzgv_lJIn++Yh46 z=0WE3c}KZPbhfCXkLCBA95NG~Pl+d!6sGGkG&cX5v7|$0^1CX=(;Lr)Sgvn}IP_JL zGY{yKkk18UEsgJTZz8+~5SuxoUvOcek~Tr8z<2Wx_pnLL^^`-8vHiS{d8%sTmDA$q zW3G)^hV?EK=vv=x9$yrj-5KFu-f#NZH#=@NF2vA~zA5H2Rnqq8d8c|J8|nPS<;jNi ziTk>u&*WKoW~BCN`M+l|S51()g=8>gEY9X?{YM@$^D%L65O;z#g_&eib#LiElx{k4 z4r$uxjY6tAous})HToTk;9y}}nu{fwL9#sNDT%`Ibf;+k_rY8*P$8Gcwm?;Ik)w}% ze1jzOf_J&c2hyz0D%u>DO=b5l*Y&;5H*0`l0T2*5?|ds|fUE@~!ZrLXH4$$uIm@ol z`FynTzu-rBd3AGY^L9HjJ9SM&rfpc|_NZJPo}V=|hu?F5U=oj(>&&hAI_c#}9xb7d z-@G*^!B$(t*C;{FNZrjw#sIN_&+e`UYx1LrHrpvrL8#dkTyD*1lRKDEjnpLSK3SA; zJ%;NZeLV8_mwtMaSddz{=Yn6FZ0wdFC5j3D3^A(g#f+&`7Ft%|`p(@L9lxI_7D;W< zaa3~MDIBXgNYz}9OZCvW_IjuaEATlp3K>%nh5U5WcSj_J8t$S z9l^10S?0vxom8{kGlhCzivHGC@sF?y+<8oEqIHv-uea6=>eP0zbXUi0AdhLvvAa1| z_UV?Z}v%`5m*rqf7_O>(?Xt$fgZ-3YD$EVMYE||`|>(Di z9;O>7lB#o*tVlI}bLH11E3LP{rXmnpskj-ga_{@EZxaW1H+KB$iLs06eJ0lP`gA8{ zlcW3mwK~X#lKGd1x-B*!^lG0Cs6#E_fh(+NKg5SzgIv}56F202!J{N}?k-HCv(hF) z<>!@_`z+Q!%XCJi=e(oFDRgI#_fkVV!zwQVQFg%-c(;gMG2C9Dx;hd}u63JJM``S& zl=xdV# z$Mtt~rOR zZlcLAagAr!fX+05W7ldK44o@Lq$}u44-I@M-hA}~damMiE_G8rE3Up~^TOj@vl?5XejYV!nc*C=8Sp9eRzuvcTKq|I%zP7hd;T<%lnR3#VTW!Q z_NhGm`h`|WbHlBq)*4ENa_IGZD*lPYDPtw(@CJa6`><^-+K559G>2#h9`WhkP#ZLT z*1ewd%Vo^h_EpO{xYud^&ZgeWZ&X(zBG|h4w#=(8y{gA3_|Hg;6_@gBC&-vrYjvUD zB6?Fa(_SlFq^N~IUP0CZSR1ah9t%q4WyPs>jRx^M{$V_p8|I_gCbou8Xe1c+KD0lc z`|&wluSoo0jygL*ViZ(3gBY^fYS#A$Ig$SbVPbZ)jA&{Aw~aMd$mKb=e#eW}W^1GM zBgaI+rQ?MCB7EB1j|e7e6@o`pKj(}D!xlPH7G^ro3G8e0^&IIBMF%c67INW;4d4OT zW0>dh=FJMKrGVT(M9lMqkB_j^kMb*c6?|smFSQufHnD#iB^xq<-virqld3xUdMawz z?jSZ)bz#j>ZeO-QV$A45l^rj@v5VMOo4+kf*>SwP(&As#EC6ReEA5G9;sxxD%{^y% zlWwaVZSj^@mZzFaW|9VmzVXKQ>*B+i*w1gdTKr#N1rDzN`V2N50D_14v$^84T>C~z zi&`=yG@h^ZW50z6Fk#!Sd5=@#jB5^UEhH5n`0i56p@s`cTR_S+6R?B*@@{rEtzvf5$35vRYQ%{4&t&Oyks0agXll%7k0xkmGZ8>wX5v@Mp3Nz8i^+P2Lyu!hn zFJiiz*S=clREEeCPm*rn^%hvYzuRLw*{7hdxFYsW?ualE=R75SPQ}qM37BFwo}e-l zv(JEegRbs7eR;=X++M{+{$0EaI<5Y832nq^(RgNcG12o~obx%#paPDF-vn7#9D4FRhY< zSkz?Ex0ly?iu;X+AWys!hU@G<0U2g7IFU1iPo`$%ggn{E!5|X+lKn}ps31G2q6x zQBk-C9>F6u`~F2-Ei&Z{M{+A1D}88WlYUTC~gIR-jRIPa}x6}NpD zU5W7O^#3XhWM*!L3QHN3!(~s8<-#Iu3R^MpgI`z3E}as+N73zq@M!2G6N@Hf>7DV& z!vqud7h{HB0=S@FClb_H)&4k~%Ci9Fcdt7-O2MYj)LJDyZ4d3r8bEqPi^tuUe;c&e z1kQZs9AN@%gUw2dv#E+CautRx@Q;QLZ~`R}Er|30pMks0qp-t}k28CCrnH@Gh4a4% z6=oS{l>35qfb0NKrvQ-QC@<~kX$_)cVw;iTuL#C@)p=qt^dhGWr68XY%7?LK&^OaY z00*T2w)l52|IVmXYa0lV)e51REpV-f;P~>g89D}5cCA1#8cGxWdZWEO0Eg)}#uY~= zY)=#wu1QE9_*si_E_?I!5t;LcTBE@%*6Kn-dmRrzWGH0)5rA&-E_(?T9O(1{k&eYp zMbLijj`QjuPbs=(9^8=tXn?j=GO|7xynkiEjRJ28?BRP)FJOto%h5K8$=>zG=Xo_{ zok+?Y)HYOiUo-WQW8BV|;fn07`Qfh;r;eRYRY*MorSI{&(wRzZIZWUP>Jl(DU^SFdj;&v4P)OB4Un0N8327^)^Wz(AAuJRy%Pg(~gXA93BS%PGI!D?T6#obUUm zelD70OZe4fwmN;e4*VwXtkFv!?8^b5d-G-iut_0n$yMwP9M74lKETi z?@crb`U{|}Ahg*)2K;P)e!1*Hhq^Mo{>D;?G<3y4qp1-4F4H@YlZp>!6*%K2LBP70fxqX zk$zyXVRN{s@1fdkM6q2VoTe$A`&+vH>mU|?rr1l@Ac;n~yD~8J8|+(^A*XF$Yt+ie z1d$!;i|QRvnv~QE2MVYfB@I`12~jHJzj#EUnSO-@z^-g(jShCQ9W_+0Wlw_@e|c@| zAZUjVt%=X@zeqAECFyL`I`U7`u<&azRJp{93Y%q7 z@@VcP9SscN`jbbb>NtqauV~UW*JBqiyTt3y@e2&~@3|dx`2*Z^u&94nV(r96qS?87 zLwyHh`g*ekMXIq1A9&l;%#-L6L6VZvo{M@3Abhm0({#{~Qx>+qJ*40Ls>yCShId60 zhIaKc%I<7DgrXeoDA#m2eD02MgkdB9o53sIljLntswR2dztRLLRQk#itolS-v^b|Q? zTW0_WcwtAE--C8#5Kb4@Y*3n8b35&P3IH6%{lm^6oxz9MpDxQ|O=qTfZ~LY-4VDg1 z=LeX;`&;}|1%s~u499>SpbZdC;RcoED2%@N;3wfdRPQMOKoXeDjT8Uy5 zQl=}ZRS}?!g1YOmCSo1B?J2XRvny;e6CVBc9&@gb0GiwRgvP$HUXVw&p-!=fu10I0$Nf~DQ znQ(Q%W5FoHfPPOB_N(8h@ikp1SfRlG>#1?FG4nAHwxmnzya%#Zmu?d*8b>Aq*Z7t` z{dZpDhbXQiWgtX6DZi<4nmei$_NVo&6)$*DtH5s|y|D3-)^7hNcSAP9&7dpjNiq!UD5o(8oWbB< zy5CBX^1ssd?vV72-px+0(d{LHp`PxSYK*n%%gs!lPd{Y@zTi7x05h4O}Nxi_nHF}lf3X7 zCW!#2|Ln#kBlhkCI0w1ZSlIs7hxgW9-!U#%(Wf&KpEo7ezhqOt0QfjmSLV#|<7CKu zCFj84n}|Bfom{KV^R+wQP6MxOgAqKxz5-Ttv5l8!`U(Ua6HRb3?Dq?Hl;M_$%jP>E zGY1k-SgJj-Zj93^Uy4K$cd^0k&WArpi=K)$g-t3jPx9v{vT*QuMmcaY)shACsJ1KO zQ4rVebg_azjCR&69CRmzy6)0PpN9OqqL8`IPJ*j-j;usEp(yGNkfPe|2@VwIp2nmhiM*{-D$c=vOK8 zz@GIG5fhMOxzw^qv)z)Stqnk>M1767qE%h2o4jfny{=%9{AJVnfD4H?Y%CMpur_H5LQ0WbM=Kf&`=d^=d#TbCWQ&OEMQIrN%cNfm ze5Q00r=%zP1_l>UwwnwX+udojy8mtK#Q%!{I+5+1*7|SxlZ-o$eNgv=@3cxvCD?;m z4`R1-jCT(``m4cmg2<(+PX&DUfDG6(nBlrl&j{rJcHwj5N8bWnlaPps6^hx(TF6ww z-u*aow^_M%R+P)c;9kl*6_%cBrf;A}lxoAlqFqf}rNH#6E)M#{@4-0qP+ehOMaY%o z)5Gz16278u0?h33XC$5v5Wo0NydCXx*u4eDR2wASr_SJ}uc9q&E ze7R4S-nrA-U#Ci*wn=9aLrJZde2;VN1y{n--YD`6Rnbo7#Ep{YcBb!h=GP~2V?%OI zC`5^9D8l;neDkQ=VeF;G8y8}(bu;iN9Z^<08XWDdo)3tB8qyjl+kV`LLcWQL%(=M! z&#vxb{L}A|)Ux^B%)gE9A+f`7sfDBIwNeJuyVA~UL7mOLLC9bB!k_s21mr`FS*24jS^?eT}P5b(+UA(G5U_w7#QPhINM_=&qhh{U=w zZ_6459;wH9+G3@SPLQng&S@1XYD%^^2OVWIL0L>$n4)m7v4+qB&dECaD~}HMf3mw< zhDimrB`M6Wf$B$Z#ho4`#S*!>VXAJYel{dk0yZcdI1bz05zW_S*q%(sv`kTaiU;Wt^YSh6u+7dgC6w_=EzLFj4Rvg5T*$)1`vzXO2x3w59 zC-m5oD*iE9z>mZ;;^wy(vX`g(c8cA)YD6)Do+M zEoA$d0CQC!f^YHgAk|TLJpk6oI2>%9;a1P;c@jXKE=gmS1M1?XN1+YbCDnaK9#u9A zJ&vmhBCW{7Ln(JSJ>sb=lWZ9NeIH+XR7bFHN)qC@HG+h%0nyWY9cUa}?R(6op4H6M z-LX?v;xLw7cIf0!P&?XBIqQ#DTTB)KVDz#~M7L!HA zXEPEq`%huF3C}5ssp2z;9L{dO0loL^rU`M+WF$V4Mh(t2>WYGsyi*Azz&zyEgsl?NVi9n+vQS5%4q>0F{ z+HTy9A$ohWrwt&CEr3~GSo+Ex z1r$9tE6iyP8gts-@q2oqC7OJ>w(u41Ofx7S|Lf|28qi`|zZsai8qs!9*%-ZX@l(2a zSAx>d&!~rj7AFXg&otc1uUoj}%TLIAa3HmD`P*YEx={7SE!N)>>)1X|w?eV3ka#^A zM|0R*aZjd&5TDY*g)?Ag``h0Ul#NOQ78tQEc-a7kqT1{Ty8m0t}C^L zQz|V6n^?LK4PdYW9Ab!uGU(S}`ni-xzl@ARovbtQ_Euh3_NT_FEnMG!)7Oq@a030r zIdP07#uarvcy?B$Bh!E!yE9&HYyCriZdfa-VN-*T^Z3nI=FGsokRzJk!haip^J)1v zp2{&K`d)P2K=*THZ)l5deQN}DTUcq$>D^W6*5KPewa(w8rK`dM#g_FK@RS>{%-8c9 z7nf=uaaaO{i~5xpQG@0n5OXU!G?>!3SNXu6(Jw9KD#f;y2@q&dy!_x^QfbEwGuytR zk4ebM4y5eQKDWUzmz?x9ahX0&Ip>MK&<>UIWkxX|uK&$%f~Sh)Bj`Vt+?g=x(9CH` z1Aur8`|ZHa`Uj%$$h9R(;oPr9Yg_QqeCUa(LrVy-i9O#Hx3=Cs55yCV<1Bwms%1^3 z-C6mI@EYf+rR8%Y98ra|QSz-G9x5KoXJKk?!EIqqyM16F5A%$MsnLV2W)chMY# zS~d4@a#*{1{pUL)R!gP(j0@IktE)S^^w@u*q1gR@MI(o+gQEZrZaW(AXlm%kv+Li& zYRH+w$LrQ-Hvlx31bV+cf0rGTg>2A4DM0&;{6=PSoMl+A$b@HvZ>y@D4MNK~MICO9 z<^uZT3lP(-5^7%USG!a5b)6`rC7EJ4@=K(GPQ__1KfkxkY_wI>%Yw~g8F_&*%rilU0$0 zEYQY#B0CSpPf~>@$mEjyV!ZU+KuV7MfNFkmyA32-!5?(O+e5{)jpo0u&k&5wzwKKM zOYJuJXH?pAEWc}~2_jr4?-2s{tK`SN@ORp$HUd5D!$Of-But-vuRG@3{<_$LLC)_G&-ze?=a;V9PF6+8E|&GvodzaJfo`4%#$!xfb6h z3m6Fy$&p1ICh^@jM1Jux`WTU8!;c2`nZ%hZ5>mkOsE(rS>Dhii>P|CxnouzxzQPKW zZ(QDMu~0d$F87}*Ln^HfrB?5?X8S4ToWM(wODpmKABxDj-j}%B`~h()=YwT{w}nZu zxrmOqt46CWF4ctoY`T~o9{=IfS4!5W1olgBdL>P3E?wC8{&|}1h^Lhv3im^xkxLap zSu@j0b3NsStM@BR9lf(%%*`TD9ezGX)-z?8y}dA>=>VJ8ofqG}goK>Pt3e3*mdcRn z#siD=DbU?jRZ+%|OPHmd@&5gvJK5wo1Bnl&z>P2cC&^EcFzb9oZ*$0!tLs`P#5yGx z?dT=875mW9x>YRgp>F>-!#LY2|0BHXO4E+*mlYHl_=!EBDl^BPga7 zynKS(0WXg3!6ki-+YtzBzL9VmLE%em2m^P!g%L7`T*ufGqf#$jafX` z-0BMWjtkglR=udj_}x7@orMK(9HTnp|6f4g@-;~4@x7;_$`x)fhywk`_7qqz>XCwWc?f^_q$dpLxr*mK1tzQ zEKyb2J0bZ~jAx)xp6vBRIZeze8*tMNxE2|zSQ!oluJx%uQ&@(9mPylpE}cP3x?&>J zI|hVRnmJ4a`D7U7-aZ0W%>S{;0E*BIiQ`L9RUoQ@g*W>3 zNe+9@S1q=LiW|&Kw?sOb1+QF8n&JAMH$>8Y$V40d^H}F)Z}a(KB4@Lwy|%3YYTq z=&`Zz#nKNJ?iIBE7ngefx5oc8@IMVqkk8oe(pc#HiJYJIvHPnJUCpPD(Er#*|390e B<>vqZ literal 0 HcmV?d00001 From 4ba649ae2300a1bb21781e9a070d1d8e9529eaf5 Mon Sep 17 00:00:00 2001 From: Omer Levi Hevroni Date: Sun, 9 Feb 2020 14:44:08 +0200 Subject: [PATCH 234/257] Update getting-started.md (#2111) re-order blog posts Signed-off-by: omerlh --- docs/getting-started.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 45a375a51a..f291822849 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -106,13 +106,10 @@ We also have example Grafana dashboards [here](/examples/dashboards/dashboards.m ## Blog posts -* 2018: +* 2020: - * [Introduction blog post](https://improbable.io/games/blog/thanos-prometheus-at-scale) - * [Monzo user story](https://monzo.com/blog/2018/07/27/how-we-monitor-monzo) - * [Banzai Cloud hand's on](https://banzaicloud.com/blog/hands-on-thanos/) - * [uSwitch user story](https://medium.com/uswitch-labs/making-prometheus-more-awesome-with-thanos-fbec8c6c28ad) - * [Thanos usage](https://www.infracloud.io/thanos-ha-scalable-prometheus/) + * [Banzai Cloud user story](https://banzaicloud.com/blog/multi-cluster-monitoring/) + * [A Production Thanos Deployment](https://www.omerlh.info/2020/02/08/a-production-thanos-deployment/) * 2019: @@ -124,9 +121,13 @@ We also have example Grafana dashboards [here](/examples/dashboards/dashboards.m * [Taboola user story](https://engineering.taboola.com/monitoring-and-metering-scale/) * [Thanos via Prometheus Operator](https://kkc.github.io/2019/02/10/prometheus-operator-with-thanos/) -* 2020: +* 2018: - * [Banzai Cloud user story](https://banzaicloud.com/blog/multi-cluster-monitoring/) + * [Introduction blog post](https://improbable.io/games/blog/thanos-prometheus-at-scale) + * [Monzo user story](https://monzo.com/blog/2018/07/27/how-we-monitor-monzo) + * [Banzai Cloud hand's on](https://banzaicloud.com/blog/hands-on-thanos/) + * [uSwitch user story](https://medium.com/uswitch-labs/making-prometheus-more-awesome-with-thanos-fbec8c6c28ad) + * [Thanos usage](https://www.infracloud.io/thanos-ha-scalable-prometheus/) ## Integrations From 43ba9c4d9a41719953df12fe559d5bb15eb5df07 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 11 Feb 2020 08:30:48 +0000 Subject: [PATCH 235/257] Fixed design doc statuses. (#2119) Signed-off-by: Bartlomiej Plotka --- docs/proposals/201809_gossip-removal.md | 2 +- docs/proposals/201909_thanos_sharding.md | 2 +- docs/proposals/201912_thanos_binary_index_header.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/proposals/201809_gossip-removal.md b/docs/proposals/201809_gossip-removal.md index 91a0de97c8..5c0b453ccb 100644 --- a/docs/proposals/201809_gossip-removal.md +++ b/docs/proposals/201809_gossip-removal.md @@ -2,7 +2,7 @@ title: Deprecated gossip clustering in favor of File SD type: proposal menu: proposals -status: completed +status: complete owner: bwplotka --- diff --git a/docs/proposals/201909_thanos_sharding.md b/docs/proposals/201909_thanos_sharding.md index 55e8836722..b033707568 100644 --- a/docs/proposals/201909_thanos_sharding.md +++ b/docs/proposals/201909_thanos_sharding.md @@ -2,7 +2,7 @@ title: Thanos Sharding for Long Term Retention Storage type: proposal menu: proposals -status: accepted +status: complete owner: bwplotka --- diff --git a/docs/proposals/201912_thanos_binary_index_header.md b/docs/proposals/201912_thanos_binary_index_header.md index 5f870f78c4..69c3158aa1 100644 --- a/docs/proposals/201912_thanos_binary_index_header.md +++ b/docs/proposals/201912_thanos_binary_index_header.md @@ -2,7 +2,7 @@ title: Binary Format for Index-cache; Renaming to Index-header. type: proposal menu: proposals -status: proposed +status: complete owner: bwplotka --- From 573eff4c205c7369edc71d785af0b251e1bfd49e Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Tue, 11 Feb 2020 17:38:17 +0100 Subject: [PATCH 236/257] bucket: Set state of status prober properly (#2120) * Set state of status prober proberly Signed-off-by: Kemal Akkoyun * Add bucket web to quickstart script Signed-off-by: Kemal Akkoyun * Add changelog Signed-off-by: Kemal Akkoyun --- CHANGELOG.md | 1 + cmd/thanos/bucket.go | 2 ++ scripts/quickstart.sh | 11 +++++++++++ 3 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f7a8d9066..9a7737f4d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#2015](https://github.com/thanos-io/thanos/pull/2015) Sidecar: Querier /api/v1/series bug fixed when time range was ignored inside sidecar. The bug was noticeable for example when using Grafana template variables. +- [#2120](https://github.com/thanos-io/thanos/pull/2120) Bucket Web: Set state of status prober properly. ## [v0.10.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0) - 2020.01.13 diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index a6a5cb771b..1aed48e781 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -355,6 +355,8 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str } g.Add(func() error { + statusProber.Ready() + return refresh(ctx, logger, bucketUI, *interval, *timeout, name, reg, objStoreConfig) }, func(error) { cancel() diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh index ccdee968b6..bde47d9bfd 100755 --- a/scripts/quickstart.sh +++ b/scripts/quickstart.sh @@ -212,4 +212,15 @@ for i in $(seq 0 1); do ${STORES} & done +sleep 0.5 + +if [ -n "${GCS_BUCKET}" -o -n "${S3_ENDPOINT}" ]; then + ${THANOS_EXECUTABLE} bucket web \ + --debug.name bucket-web \ + --log.level debug \ + --http-address 0.0.0.0:10933 \ + --http-grace-period 1s \ + ${OBJSTORECFG} & +fi + wait From 508d388652810a6f39382f47b5dfcc731fbc6f8a Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Wed, 12 Feb 2020 16:18:19 +0000 Subject: [PATCH 237/257] Cloose index-header and block properly, even on error case. (#2125) Especially for binary index-header we were not closing mmaped file handle. Signed-off-by: Bartlomiej Plotka --- pkg/store/bucket.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index e7de6cdc40..0ce8f97402 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -313,10 +313,7 @@ func (s *BucketStore) Close() (err error) { defer s.mtx.Unlock() for _, b := range s.blocks { - if e := b.Close(); e != nil { - level.Warn(s.logger).Log("msg", "closing Bucket block failed", "err", err) - err = e - } + runutil.CloseWithErrCapture(&err, b, "closing Bucket Block") } return err } @@ -462,6 +459,11 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er return errors.Wrap(err, "create index cache reader") } } + defer func() { + if err != nil { + runutil.CloseWithErrCapture(&err, indexHeaderReader, "index-header") + } + }() b, err := newBucketBlock( ctx, @@ -478,6 +480,12 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er if err != nil { return errors.Wrap(err, "new bucket block") } + defer func() { + if err != nil { + runutil.CloseWithErrCapture(&err, b, "index-header") + } + }() + s.mtx.Lock() defer s.mtx.Unlock() @@ -1216,11 +1224,10 @@ func newBucketBlock( } // Get object handles for all chunk files. - err = bkt.Iter(ctx, path.Join(meta.ULID.String(), block.ChunksDirname), func(n string) error { + if err = bkt.Iter(ctx, path.Join(meta.ULID.String(), block.ChunksDirname), func(n string) error { b.chunkObjs = append(b.chunkObjs, n) return nil - }) - if err != nil { + }); err != nil { return nil, errors.Wrap(err, "list chunk files") } return b, nil @@ -1279,7 +1286,8 @@ func (b *bucketBlock) chunkReader(ctx context.Context) *bucketChunkReader { // Close waits for all pending readers to finish and then closes all underlying resources. func (b *bucketBlock) Close() error { b.pendingReaders.Wait() - return nil + + return b.indexHeaderReader.Close() } // bucketIndexReader is a custom index reader (not conforming index.Reader interface) that reads index that is stored in From 254aeee8d2782bc1df63fcb5516d046f7a0d2942 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 13 Feb 2020 03:52:59 -0500 Subject: [PATCH 238/257] fix store dashboard metric name (#2127) Signed-off-by: yeya24 --- examples/dashboards/store.json | 12 ++++++------ mixin/thanos/dashboards/store.libsonnet | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/dashboards/store.json b/examples/dashboards/store.json index 4b0f0aaba9..13d3e4b9df 100644 --- a/examples/dashboards/store.json +++ b/examples/dashboards/store.json @@ -2533,7 +2533,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "P99 {{job}}", @@ -2541,7 +2541,7 @@ "step": 10 }, { - "expr": "sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "expr": "sum(rate(thanos_bucket_store_series_merge_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_bucket_store_series_merge_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "mean {{job}}", @@ -2549,7 +2549,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(thanos_bucket_store_series_merge_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "P50 {{job}}", @@ -2626,7 +2626,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "P99 {{job}}", @@ -2634,7 +2634,7 @@ "step": 10 }, { - "expr": "sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", + "expr": "sum(rate(thanos_bucket_store_series_gate_duration_seconds_sum{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job) * 1 / sum(rate(thanos_bucket_store_series_gate_duration_seconds_count{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "mean {{job}}", @@ -2642,7 +2642,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(thanos_bucket_store_series_gate_duration_seconds_bucket{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job, le)) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "P50 {{job}}", diff --git a/mixin/thanos/dashboards/store.libsonnet b/mixin/thanos/dashboards/store.libsonnet index 2f8d27d219..756578989b 100644 --- a/mixin/thanos/dashboards/store.libsonnet +++ b/mixin/thanos/dashboards/store.libsonnet @@ -234,11 +234,11 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; ) .addPanel( g.panel('Merge', 'Shows how long has it taken to merge series.') + - g.latencyPanel('thanos_bucket_store_series_merge_duration_seconds_bucket', 'namespace="$namespace",job=~"$job"') + g.latencyPanel('thanos_bucket_store_series_merge_duration_seconds', 'namespace="$namespace",job=~"$job"') ) .addPanel( g.panel('Gate', 'Shows how long has it taken for a series to wait at the gate.') + - g.latencyPanel('thanos_bucket_store_series_gate_duration_seconds_bucket', 'namespace="$namespace",job=~"$job"') + g.latencyPanel('thanos_bucket_store_series_gate_duration_seconds', 'namespace="$namespace",job=~"$job"') ) ) .addRow( From 4cc8442a566e00313e4eef60b4e8e409d114f536 Mon Sep 17 00:00:00 2001 From: Dominic Green Date: Thu, 13 Feb 2020 09:09:23 +0000 Subject: [PATCH 239/257] removing myself as a maintainer :( (#2129) Signed-off-by: Dominic Green --- MAINTAINERS.md | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 08ad64261d..c36e87defa 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -3,7 +3,6 @@ | Name | Email | Slack | GitHub | Company | |-----------------------|------------------------|--------------------------|---------------------------------------------|-------------------| | Bartłomiej Płotka | bwplotka@gmail.com | `@bwplotka` | [@bwplotka](https://github.com/bwplotka) | Red Hat | -| Dominic Green | dom@improbable.io | `@domgreen` | [@domgreen](https://github.com/domgreen) | Improbable | | Frederic Branczyk | fbranczyk@gmail.com | `@brancz` | [@brancz](https://github.com/brancz) | Red Hat | | Giedrius Statkevičius | giedriuswork@gmail.com | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS) | AdForm | | Povilas Versockas | p.versockas@gmail.com | `@povilasv` | [@povilasv](https://github.com/povilasv) | Utility Warehouse | From 811379629c707734d639adc17c2ffba979893896 Mon Sep 17 00:00:00 2001 From: khyatisoneji <31898080+khyatisoneji@users.noreply.github.com> Date: Thu, 13 Feb 2020 16:57:40 +0530 Subject: [PATCH 240/257] proposal: Updated RW proposal (#2131) Signed-off-by: khyatisoneji --- .../201901-read-write-operations-bucket.md | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/proposals/201901-read-write-operations-bucket.md b/docs/proposals/201901-read-write-operations-bucket.md index bb58acbe79..4658e095e7 100644 --- a/docs/proposals/201901-read-write-operations-bucket.md +++ b/docs/proposals/201901-read-write-operations-bucket.md @@ -138,7 +138,18 @@ manual actions. This is on purpose to not allow block malformation by blocks whi *Newer block from two or more overlapping blocks fully submatches the source blocks of older blocks. Older blocks can be then ignored.* -The word **fully** is crucial. For example we won't be able to resolve case with block ABCD and CDEF. This is because there is no logic for decompact or vertical compaction. +The word **fully** is crucial. + +We will determine overlaps in data by using **Full Overlap Block Detection Algorithm** which is defined as following: +Consider the below diagram which shows different tsdb blocks: + +[![Diagram](https://docs.google.com/drawings/d/e/2PACX-1vTVX4WgIa4O0rcvN8R_oZJBDOGNe0-eTJW7Ucqw0CnwcjuswATupMRD3r97b94cW6xasVsU7MPQPpVf/pub?w=950&h=720)](https://docs.google.com/drawings/d/1RsP7q2JPUrOZG_6uDAJo4D5hWM9zjqV0wHTLPQchpKE/edit?usp=sharing) + +In the first example, we see that the block F is a compacted block containing data of blocks ABCD. The blocks ABCD are source blocks that contain data of ABCD blocks. +Since block F contains data for all blocks that blocks ABCD contain, the block F completely overlaps with blocks ABCD, we can safely delete ABCD. + +In the second example, the block F is a compacted block containing data of blocks ABCD. The blocks ABCD are blocks contains data of ABCDE blocks. +Since block F doesn't contain data related to block E, the block F cannot be safely deleted. Having this kind of overlap support, we can delay deletion by forming 6th rule: @@ -162,17 +173,23 @@ To match partial upload safeguards we want to delete block in reverse order: > 7 . To schedule delete operation, delete `meta.json` file. All components will exclude this block and compactor will do eventual deletion assuming the block is partially uploaded. [Compactor change needed] We schedule deletions instead of doing them straight away for 3 reasons: + * Readers that have loaded this block can still access index and metrics for some time. On next sync they will notice lack of meta.json and assume partial block which excludes it from block being loaded. * Only compactor deletes metrics and index. * In further delete steps, starting with meta.json first ensures integrity mark being deleted first, so in case of deletion process being stopped, we can treat this block as partial block (rule 4th) and delete it gracefully. +Along with this, we also store information about when the block was scheduled to be deleted so that it can be deleted at a later point in time. +To do so, we create a file `compactor-meta.json` where we store information about when the block was scheduled to be deleted. +Storing the information in a file makes it resilient to failures that result in restarts. + There might be exception for malformed blocks that blocks compaction or reader operations. Since we may need to unblock the system immediately the block can be forcibly removed meaning that query failures may occur (reader loaded block, but not aware block was deleted). > 8 . Compactor waits minimum 15m (`deleteDelay`) before deleting the whole `To Delete` block. [Compactor change needed] -This is to make sure we don't forcibly remove block which is still loaded on reader side. We do that by counting time from -spotting lack of meta.json first. After 15 minutes we are ok to delete the whole directory. +This is to make sure we don't forcibly remove block which is still loaded on reader side. + +We check the `compactor-meta.json` file to identify if the block has to be deleted. After 15 minutes of marking the block to be deleted, we are ok to delete the whole block directory. ## Risks From 5e877225a625b10de9c12eb9e2e14406a88e8698 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Thu, 13 Feb 2020 13:32:44 +0100 Subject: [PATCH 241/257] Fix broken make task (#2132) Signed-off-by: Kemal Akkoyun --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8f7ee0c7a7..780bae1b7b 100644 --- a/Makefile +++ b/Makefile @@ -362,9 +362,9 @@ JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-st .PHONY: jsonnet-format jsonnet-format: - @which jsonnetfmt 2>/dev/null || ( \ - echo "Cannot find jsonnetfmt command, please install from https://github.com/google/jsonnet/releases. If your C++ does not support GLIBCXX_3.4.20, please use xxx-in-container target like jsonnet-format-in-container." && exit 1 - ) + @which jsonnetfmt 2>&1 >/dev/null || ( \ + echo "Cannot find jsonnetfmt command, please install from https://github.com/google/jsonnet/releases.\nIf your C++ does not support GLIBCXX_3.4.20, please use xxx-in-container target like jsonnet-format-in-container." \ + && exit 1) find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \ xargs -n 1 -- $(JSONNET_FMT) -i From adfef4b58f176a514f734751bd7cace7e57ecef6 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Thu, 13 Feb 2020 13:57:08 +0100 Subject: [PATCH 242/257] Update jb and fix issues (#2133) Signed-off-by: Kemal Akkoyun --- Makefile | 27 +++++++-------- jsonnetfile.json | 33 ------------------- mixin/jsonnetfile.json | 32 ++++++++++++++++++ .../jsonnetfile.lock.json | 27 ++++++++------- 4 files changed, 59 insertions(+), 60 deletions(-) delete mode 100644 jsonnetfile.json create mode 100644 mixin/jsonnetfile.json rename jsonnetfile.lock.json => mixin/jsonnetfile.lock.json (81%) diff --git a/Makefile b/Makefile index 780bae1b7b..d26d2a88e2 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ GOJSONTOYAML ?= $(GOBIN)/gojsontoyaml-$(GOJSONTOYAML_VERSION) # v0.14.0 JSONNET_VERSION ?= fbde25be2182caa4345b03f1532450911ac7d1f3 JSONNET ?= $(GOBIN)/jsonnet-$(JSONNET_VERSION) -JSONNET_BUNDLER_VERSION ?= d7829f6c7e632e954c0e5db8b3eece8f111f9461 +JSONNET_BUNDLER_VERSION ?= efe0c9e864431e93d5c3376bd5931d0fb9b2a296 JSONNET_BUNDLER ?= $(GOBIN)/jb-$(JSONNET_BUNDLER_VERSION) # Prometheus v2.14.0 PROMTOOL_VERSION ?= edeb7a44cbf745f1d8be4ea6f215e79e651bfe19 @@ -56,7 +56,8 @@ PROMTOOL ?= $(GOBIN)/promtool-$(PROMTOOL_VERSION) # systems gsed won't be installed, so will use sed as expected. SED ?= $(shell which gsed 2>/dev/null || which sed) -MIXIN_ROOT ?= mixin/thanos +MIXIN_ROOT ?= mixin +THANOS_MIXIN ?= mixin/thanos JSONNET_VENDOR_DIR ?= mixin/vendor WEB_DIR ?= website @@ -332,31 +333,31 @@ examples-in-container: examples .PHONY: examples -examples: jsonnet-format mixin/thanos/README.md examples/alerts/alerts.md examples/alerts/alerts.yaml examples/alerts/rules.yaml examples/dashboards examples/tmp +examples: jsonnet-format ${THANOS_MIXIN}/README.md examples/alerts/alerts.md examples/alerts/alerts.yaml examples/alerts/rules.yaml examples/dashboards examples/tmp $(EMBEDMD) -w examples/alerts/alerts.md - $(EMBEDMD) -w mixin/thanos/README.md + $(EMBEDMD) -w ${THANOS_MIXIN}/README.md .PHONY: examples/tmp examples/tmp: -rm -rf examples/tmp/ -mkdir -p examples/tmp/ - $(JSONNET) -J ${JSONNET_VENDOR_DIR} -m examples/tmp/ ${MIXIN_ROOT}/separated_alerts.jsonnet | xargs -I{} sh -c 'cat {} | $(GOJSONTOYAML) > {}.yaml; rm -f {}' -- {} + $(JSONNET) -J ${JSONNET_VENDOR_DIR} -m examples/tmp/ ${THANOS_MIXIN}/separated_alerts.jsonnet | xargs -I{} sh -c 'cat {} | $(GOJSONTOYAML) > {}.yaml; rm -f {}' -- {} .PHONY: examples/dashboards # to keep examples/dashboards/dashboards.md. -examples/dashboards: $(JSONNET) ${MIXIN_ROOT}/mixin.libsonnet ${MIXIN_ROOT}/defaults.libsonnet ${MIXIN_ROOT}/dashboards/* +examples/dashboards: $(JSONNET) ${THANOS_MIXIN}/mixin.libsonnet ${THANOS_MIXIN}/defaults.libsonnet ${THANOS_MIXIN}/dashboards/* -rm -rf examples/dashboards/*.json - $(JSONNET) -J ${JSONNET_VENDOR_DIR} -m examples/dashboards ${MIXIN_ROOT}/dashboards.jsonnet + $(JSONNET) -J ${JSONNET_VENDOR_DIR} -m examples/dashboards ${THANOS_MIXIN}/dashboards.jsonnet -examples/alerts/alerts.yaml: $(JSONNET) $(GOJSONTOYAML) ${MIXIN_ROOT}/mixin.libsonnet ${MIXIN_ROOT}/defaults.libsonnet ${MIXIN_ROOT}/alerts/* - $(JSONNET) ${MIXIN_ROOT}/alerts.jsonnet | $(GOJSONTOYAML) > $@ +examples/alerts/alerts.yaml: $(JSONNET) $(GOJSONTOYAML) ${THANOS_MIXIN}/mixin.libsonnet ${THANOS_MIXIN}/defaults.libsonnet ${THANOS_MIXIN}/alerts/* + $(JSONNET) ${THANOS_MIXIN}/alerts.jsonnet | $(GOJSONTOYAML) > $@ -examples/alerts/rules.yaml: $(JSONNET) $(GOJSONTOYAML) ${MIXIN_ROOT}/mixin.libsonnet ${MIXIN_ROOT}/defaults.libsonnet ${MIXIN_ROOT}/rules/* - $(JSONNET) ${MIXIN_ROOT}/rules.jsonnet | $(GOJSONTOYAML) > $@ +examples/alerts/rules.yaml: $(JSONNET) $(GOJSONTOYAML) ${THANOS_MIXIN}/mixin.libsonnet ${THANOS_MIXIN}/defaults.libsonnet ${THANOS_MIXIN}/rules/* + $(JSONNET) ${THANOS_MIXIN}/rules.jsonnet | $(GOJSONTOYAML) > $@ .PHONY: jsonnet-vendor -jsonnet-vendor: $(JSONNET_BUNDLER) jsonnetfile.json jsonnetfile.lock.json +jsonnet-vendor: $(JSONNET_BUNDLER) $(MIXIN_ROOT)/jsonnetfile.json $(MIXIN_ROOT)/jsonnetfile.lock.json rm -rf ${JSONNET_VENDOR_DIR} - $(JSONNET_BUNDLER) install --jsonnetpkg-home="${JSONNET_VENDOR_DIR}" + cd ${MIXIN_ROOT} && $(JSONNET_BUNDLER) install JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-style s diff --git a/jsonnetfile.json b/jsonnetfile.json deleted file mode 100644 index 1bcb73e8a9..0000000000 --- a/jsonnetfile.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "dependencies": [ - { - "name": "thanos-mixin", - "source": { - "local": { - "directory": "mixin/thanos" - } - }, - "version": "." - }, - { - "name": "grafonnet", - "source": { - "git": { - "remote": "https://github.com/grafana/grafonnet-lib", - "subdir": "grafonnet" - } - }, - "version": "master" - }, - { - "name": "grafana-builder", - "source": { - "git": { - "remote": "https://github.com/grafana/jsonnet-libs", - "subdir": "grafana-builder" - } - }, - "version": "master" - } - ] -} diff --git a/mixin/jsonnetfile.json b/mixin/jsonnetfile.json new file mode 100644 index 0000000000..8f8d9d3559 --- /dev/null +++ b/mixin/jsonnetfile.json @@ -0,0 +1,32 @@ +{ + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet-lib", + "subdir": "grafonnet" + } + }, + "version": "master" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs", + "subdir": "grafana-builder" + } + }, + "version": "master" + }, + { + "source": { + "local": { + "directory": "thanos" + } + }, + "version": ".", + "name": "thanos-mixin" + } + ], + "legacyImports": true +} diff --git a/jsonnetfile.lock.json b/mixin/jsonnetfile.lock.json similarity index 81% rename from jsonnetfile.lock.json rename to mixin/jsonnetfile.lock.json index ae23cf5774..f2a1581a48 100644 --- a/jsonnetfile.lock.json +++ b/mixin/jsonnetfile.lock.json @@ -1,35 +1,34 @@ { "dependencies": [ { - "name": "grafana-builder", "source": { "git": { - "remote": "https://github.com/grafana/jsonnet-libs", - "subdir": "grafana-builder" + "remote": "https://github.com/grafana/grafonnet-lib", + "subdir": "grafonnet" } }, - "version": "f4c59f64f80442f871a06c91edf74d014b82acaf", - "sum": "ELsYwK+kGdzX1mee2Yy+/b2mdO4Y503BOCDkFzwmGbE=" + "version": "69bc267211790a1c3f4ea6e6211f3e8ffe22f987", + "sum": "BjHfWzqSAgtAKEVD6ipoYOkb8XT5wSBIboY4ZLwhlOU=" }, { - "name": "grafonnet", "source": { "git": { - "remote": "https://github.com/grafana/grafonnet-lib", - "subdir": "grafonnet" + "remote": "https://github.com/grafana/jsonnet-libs", + "subdir": "grafana-builder" } }, - "version": "69bc267211790a1c3f4ea6e6211f3e8ffe22f987", - "sum": "BjHfWzqSAgtAKEVD6ipoYOkb8XT5wSBIboY4ZLwhlOU=" + "version": "f4c59f64f80442f871a06c91edf74d014b82acaf", + "sum": "ELsYwK+kGdzX1mee2Yy+/b2mdO4Y503BOCDkFzwmGbE=" }, { - "name": "thanos-mixin", "source": { "local": { - "directory": "mixin/thanos" + "directory": "thanos" } }, - "version": "" + "version": "", + "name": "thanos-mixin" } - ] + ], + "legacyImports": false } From 00f2aadc02433265f12161c98d66c22ed87419e9 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 13 Feb 2020 16:22:11 -0500 Subject: [PATCH 243/257] fix sidecar dashboard stream metric (#2134) Signed-off-by: yeya24 --- examples/dashboards/sidecar.json | 10 +++++----- mixin/thanos/dashboards/sidecar.libsonnet | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/dashboards/sidecar.json b/examples/dashboards/sidecar.json index f0d5bbb432..fca87f47d2 100644 --- a/examples/dashboards/sidecar.json +++ b/examples/dashboards/sidecar.json @@ -864,7 +864,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(grpc_client_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)", + "expr": "sum(rate(grpc_server_handled_total{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", @@ -941,7 +941,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(grpc_client_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)\n", + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\",namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, grpc_code)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", @@ -1018,7 +1018,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "P99 {{job}} {{grpc_method}}", @@ -1026,7 +1026,7 @@ "step": 10 }, { - "expr": "sum(rate(grpc_client_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_client_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", + "expr": "sum(rate(grpc_server_handling_seconds_sum{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job) * 1\n/\nsum(rate(grpc_server_handling_seconds_count{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "mean {{job}} {{grpc_method}}", @@ -1034,7 +1034,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(grpc_client_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",grpc_type=\"server_stream\"}[$interval])) by (job, grpc_method, le)) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "P50 {{job}} {{grpc_method}}", diff --git a/mixin/thanos/dashboards/sidecar.libsonnet b/mixin/thanos/dashboards/sidecar.libsonnet index 37878c7b11..986101fa0b 100644 --- a/mixin/thanos/dashboards/sidecar.libsonnet +++ b/mixin/thanos/dashboards/sidecar.libsonnet @@ -60,15 +60,15 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; g.row('Detailed') .addPanel( g.panel('Rate', 'Shows rate of handled Streamed gRPC requests from queriers.') + - g.grpcQpsPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + g.grpcQpsPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') ) .addPanel( g.panel('Errors', 'Shows ratio of errors compared to the total number of handled requests from queriers.') + - g.grpcErrDetailsPanel('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + g.grpcErrDetailsPanel('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') ) .addPanel( g.panel('Duration', 'Shows how long has it taken to handle requests from queriers, in quantiles.') + - g.grpcLatencyPanelDetailed('client', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') + g.grpcLatencyPanelDetailed('server', 'namespace="$namespace",job=~"$job",grpc_type="server_stream"') ) + g.collapse ) From 66921a3d446e860e0392311dfc40221f3b1f11bb Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Fri, 14 Feb 2020 18:13:59 +0100 Subject: [PATCH 244/257] *: Adapt alerts and dashboards to naming conventions (#2140) Signed-off-by: Frederic Branczyk --- examples/alerts/alerts.md | 224 +++++++++--------- examples/alerts/alerts.yaml | 224 +++++++++--------- examples/alerts/rules.yaml | 42 ++-- .../{compactor.json => compact.json} | 10 +- examples/dashboards/overview.json | 86 +++---- .../dashboards/{querier.json => query.json} | 10 +- .../{receiver.json => receive.json} | 10 +- examples/dashboards/{ruler.json => rule.json} | 10 +- mixin/thanos/README.md | 24 +- mixin/thanos/alerts/absent.libsonnet | 8 +- mixin/thanos/alerts/alerts.libsonnet | 8 +- ...{compactor.libsonnet => compact.libsonnet} | 38 +-- .../{querier.libsonnet => query.libsonnet} | 46 ++-- .../{receiver.libsonnet => receive.libsonnet} | 24 +- .../{ruler.libsonnet => rule.libsonnet} | 62 ++--- ...{compactor.libsonnet => compact.libsonnet} | 28 +-- mixin/thanos/dashboards/dashboards.libsonnet | 8 +- .../{querier.libsonnet => query.libsonnet} | 34 +-- .../{receiver.libsonnet => receive.libsonnet} | 22 +- .../{ruler.libsonnet => rule.libsonnet} | 30 +-- mixin/thanos/defaults.libsonnet | 24 +- .../{querier.libsonnet => query.libsonnet} | 14 +- .../{receiver.libsonnet => receive.libsonnet} | 16 +- mixin/thanos/rules/rules.libsonnet | 4 +- 24 files changed, 503 insertions(+), 503 deletions(-) rename examples/dashboards/{compactor.json => compact.json} (99%) rename examples/dashboards/{querier.json => query.json} (99%) rename examples/dashboards/{receiver.json => receive.json} (99%) rename examples/dashboards/{ruler.json => rule.json} (99%) rename mixin/thanos/alerts/{compactor.libsonnet => compact.libsonnet} (60%) rename mixin/thanos/alerts/{querier.libsonnet => query.libsonnet} (68%) rename mixin/thanos/alerts/{receiver.libsonnet => receive.libsonnet} (84%) rename mixin/thanos/alerts/{ruler.libsonnet => rule.libsonnet} (63%) rename mixin/thanos/dashboards/{compactor.libsonnet => compact.libsonnet} (91%) rename mixin/thanos/dashboards/{querier.libsonnet => query.libsonnet} (90%) rename mixin/thanos/dashboards/{receiver.libsonnet => receive.libsonnet} (93%) rename mixin/thanos/dashboards/{ruler.libsonnet => rule.libsonnet} (90%) rename mixin/thanos/rules/{querier.libsonnet => query.libsonnet} (90%) rename mixin/thanos/rules/{receiver.libsonnet => receive.libsonnet} (90%) diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index e9b7da3017..3ae95fce3a 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -4,57 +4,57 @@ Here are some example alerts configured for Kubernetes environment. ## Compaction -[embedmd]:# (../tmp/thanos-compactor.rules.yaml yaml) +[embedmd]:# (../tmp/thanos-compact.rules.yaml yaml) ```yaml -name: thanos-compactor.rules +name: thanos-compact.rules rules: -- alert: ThanosCompactorMultipleCompactorsAreRunning +- alert: ThanosCompactMultipleRunning annotations: - message: You should never run more than one Thanos Compactor at once. You have - {{ $value }} - expr: sum(up{job=~"thanos-compactor.*"}) > 1 + message: No more than one Thanos Compact instance should be running at once. There + are {{ $value }} + expr: sum(up{job=~"thanos-compact.*"}) > 1 for: 5m labels: severity: warning -- alert: ThanosCompactorHalted +- alert: ThanosCompactHalted annotations: - message: Thanos Compactor {{$labels.job}} has failed to run and now is halted. - expr: thanos_compactor_halted{job=~"thanos-compactor.*"} == 1 + message: Thanos Compact {{$labels.job}} has failed to run and now is halted. + expr: thanos_compactor_halted{job=~"thanos-compact.*"} == 1 for: 5m labels: severity: warning -- alert: ThanosCompactorHighCompactionFailures +- alert: ThanosCompactHighCompactionFailures annotations: - message: Thanos Compactor {{$labels.job}} is failing to execute {{ $value | humanize + message: Thanos Compact {{$labels.job}} is failing to execute {{ $value | humanize }}% of compactions. expr: | ( - sum by (job) (rate(thanos_compact_group_compactions_failures_total{job=~"thanos-compactor.*"}[5m])) + sum by (job) (rate(thanos_compact_group_compactions_failures_total{job=~"thanos-compact.*"}[5m])) / - sum by (job) (rate(thanos_compact_group_compactions_total{job=~"thanos-compactor.*"}[5m])) + sum by (job) (rate(thanos_compact_group_compactions_total{job=~"thanos-compact.*"}[5m])) * 100 > 5 ) for: 15m labels: severity: warning -- alert: ThanosCompactorBucketHighOperationFailures +- alert: ThanosCompactBucketHighOperationFailures annotations: - message: Thanos Compactor {{$labels.job}} Bucket is failing to execute {{ $value + message: Thanos Compact {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations. expr: | ( - sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-compactor.*"}[5m])) + sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-compact.*"}[5m])) / - sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~"thanos-compactor.*"}[5m])) + sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~"thanos-compact.*"}[5m])) * 100 > 5 ) for: 15m labels: severity: warning -- alert: ThanosCompactorHasNotRun +- alert: ThanosCompactHasNotRun annotations: - message: Thanos Compactor {{$labels.job}} has not uploaded anything for 24 hours. - expr: (time() - max(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compactor.*"})) + message: Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours. + expr: (time() - max(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compact.*"})) / 60 / 60 > 24 labels: severity: warning @@ -64,107 +64,107 @@ rules: For Thanos ruler we run some alerts in local Prometheus, to make sure that Thanos Rule is working: -[embedmd]:# (../tmp/thanos-ruler.rules.yaml yaml) +[embedmd]:# (../tmp/thanos-rule.rules.yaml yaml) ```yaml -name: thanos-ruler.rules +name: thanos-rule.rules rules: -- alert: ThanosRulerQueueIsDroppingAlerts +- alert: ThanosRuleQueueIsDroppingAlerts annotations: - message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to queue alerts. + message: Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to queue alerts. expr: | - sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=~"thanos-ruler.*"}[5m])) > 0 + sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=~"thanos-rule.*"}[5m])) > 0 for: 5m labels: severity: critical -- alert: ThanosRulerSenderIsFailingAlerts +- alert: ThanosRuleSenderIsFailingAlerts annotations: - message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to send alerts + message: Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to send alerts to alertmanager. expr: | - sum by (job) (rate(thanos_alert_sender_alerts_dropped_total{job=~"thanos-ruler.*"}[5m])) > 0 + sum by (job) (rate(thanos_alert_sender_alerts_dropped_total{job=~"thanos-rule.*"}[5m])) > 0 for: 5m labels: severity: critical -- alert: ThanosRulerHighRuleEvaluationFailures +- alert: ThanosRuleHighRuleEvaluationFailures annotations: - message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to evaluate rules. + message: Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to evaluate rules. expr: | ( - sum by (job) (rate(prometheus_rule_evaluation_failures_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(prometheus_rule_evaluation_failures_total{job=~"thanos-rule.*"}[5m])) / - sum by (job) (rate(prometheus_rule_evaluations_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(prometheus_rule_evaluations_total{job=~"thanos-rule.*"}[5m])) * 100 > 5 ) for: 5m labels: severity: warning -- alert: ThanosRulerHighRuleEvaluationWarnings +- alert: ThanosRuleHighRuleEvaluationWarnings annotations: - message: Thanos Ruler {{$labels.job}} {{$labels.pod}} has high number of evaluation + message: Thanos Rule {{$labels.job}} {{$labels.pod}} has high number of evaluation warnings. expr: | - sum by (job) (rate(thanos_rule_evaluation_with_warnings_total{job=~"thanos-ruler.*"}[5m])) > 0 + sum by (job) (rate(thanos_rule_evaluation_with_warnings_total{job=~"thanos-rule.*"}[5m])) > 0 for: 15m labels: severity: warning -- alert: ThanosRulerRuleEvaluationLatencyHigh +- alert: ThanosRuleRuleEvaluationLatencyHigh annotations: - message: Thanos Ruler {{$labels.job}}/{{$labels.pod}} has higher evaluation latency + message: Thanos Rule {{$labels.job}}/{{$labels.pod}} has higher evaluation latency than interval for {{$labels.rule_group}}. expr: | ( - sum by (job, pod, rule_group) (prometheus_rule_group_last_duration_seconds{job=~"thanos-ruler.*"}) + sum by (job, pod, rule_group) (prometheus_rule_group_last_duration_seconds{job=~"thanos-rule.*"}) > - sum by (job, pod, rule_group) (prometheus_rule_group_interval_seconds{job=~"thanos-ruler.*"}) + sum by (job, pod, rule_group) (prometheus_rule_group_interval_seconds{job=~"thanos-rule.*"}) ) for: 5m labels: severity: warning -- alert: ThanosRulerGrpcErrorRate +- alert: ThanosRuleGrpcErrorRate annotations: - message: Thanos Ruler {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Rule {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests. expr: | ( - sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-rule.*"}[5m])) / - sum by (job) (rate(grpc_server_started_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(grpc_server_started_total{job=~"thanos-rule.*"}[5m])) * 100 > 5 ) for: 5m labels: severity: warning -- alert: ThanosRulerConfigReloadFailure +- alert: ThanosRuleConfigReloadFailure annotations: - message: Thanos Ruler {{$labels.job}} has not been able to reload its configuration. - expr: avg(thanos_rule_config_last_reload_successful{job=~"thanos-ruler.*"}) by (job) + message: Thanos Rule {{$labels.job}} has not been able to reload its configuration. + expr: avg(thanos_rule_config_last_reload_successful{job=~"thanos-rule.*"}) by (job) != 1 for: 5m labels: severity: warning -- alert: ThanosRulerQueryHighDNSFailures +- alert: ThanosRuleQueryHighDNSFailures annotations: - message: Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing + message: Thanos Rule {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for query endpoints. expr: | ( - sum by (job) (rate(thanos_ruler_query_apis_dns_failures_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(thanos_ruler_query_apis_dns_failures_total{job=~"thanos-rule.*"}[5m])) / - sum by (job) (rate(thanos_ruler_query_apis_dns_lookups_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(thanos_ruler_query_apis_dns_lookups_total{job=~"thanos-rule.*"}[5m])) * 100 > 1 ) for: 15m labels: severity: warning -- alert: ThanosRulerAlertmanagerHighDNSFailures +- alert: ThanosRuleAlertmanagerHighDNSFailures annotations: - message: Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing + message: Thanos Rule {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for Alertmanager endpoints. expr: | ( - sum by (job) (rate(thanos_ruler_alertmanagers_dns_failures_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(thanos_ruler_alertmanagers_dns_failures_total{job=~"thanos-rule.*"}[5m])) / - sum by (job) (rate(thanos_ruler_alertmanagers_dns_lookups_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(thanos_ruler_alertmanagers_dns_lookups_total{job=~"thanos-rule.*"}[5m])) * 100 > 1 ) for: 15m @@ -260,98 +260,98 @@ rules: ## Query -[embedmd]:# (../tmp/thanos-querier.rules.yaml yaml) +[embedmd]:# (../tmp/thanos-query.rules.yaml yaml) ```yaml -name: thanos-querier.rules +name: thanos-query.rules rules: -- alert: ThanosQuerierHttpRequestQueryErrorRateHigh +- alert: ThanosQueryHttpRequestQueryErrorRateHigh annotations: - message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests. expr: | ( - sum(rate(http_requests_total{code=~"5..", job=~"thanos-querier.*", handler="query"}[5m])) + sum(rate(http_requests_total{code=~"5..", job=~"thanos-query.*", handler="query"}[5m])) / - sum(rate(http_requests_total{job=~"thanos-querier.*", handler="query"}[5m])) + sum(rate(http_requests_total{job=~"thanos-query.*", handler="query"}[5m])) ) * 100 > 5 for: 5m labels: severity: critical -- alert: ThanosQuerierHttpRequestQueryRangeErrorRateHigh +- alert: ThanosQueryHttpRequestQueryRangeErrorRateHigh annotations: - message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests. expr: | ( - sum(rate(http_requests_total{code=~"5..", job=~"thanos-querier.*", handler="query_range"}[5m])) + sum(rate(http_requests_total{code=~"5..", job=~"thanos-query.*", handler="query_range"}[5m])) / - sum(rate(http_requests_total{job=~"thanos-querier.*", handler="query_range"}[5m])) + sum(rate(http_requests_total{job=~"thanos-query.*", handler="query_range"}[5m])) ) * 100 > 5 for: 5m labels: severity: critical -- alert: ThanosQuerierGrpcServerErrorRate +- alert: ThanosQueryGrpcServerErrorRate annotations: - message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests. expr: | ( - sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-query.*"}[5m])) / - sum by (job) (rate(grpc_server_started_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(grpc_server_started_total{job=~"thanos-query.*"}[5m])) * 100 > 5 ) for: 5m labels: severity: warning -- alert: ThanosQuerierGrpcClientErrorRate +- alert: ThanosQueryGrpcClientErrorRate annotations: - message: Thanos Querier {{$labels.job}} is failing to send {{ $value | humanize + message: Thanos Query {{$labels.job}} is failing to send {{ $value | humanize }}% of requests. expr: | ( - sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job=~"thanos-query.*"}[5m])) / - sum by (job) (rate(grpc_client_started_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(grpc_client_started_total{job=~"thanos-query.*"}[5m])) ) * 100 > 5 for: 5m labels: severity: warning -- alert: ThanosQuerierHighDNSFailures +- alert: ThanosQueryHighDNSFailures annotations: - message: Thanos Queriers {{$labels.job}} have {{ $value | humanize }}% of failing + message: Thanos Query {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for store endpoints. expr: | ( - sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-query.*"}[5m])) / - sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-query.*"}[5m])) ) * 100 > 1 for: 15m labels: severity: warning -- alert: ThanosQuerierInstantLatencyHigh +- alert: ThanosQueryInstantLatencyHigh annotations: - message: Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ $value + message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries. expr: | ( - histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"})) > 10 + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-query.*", handler="query"})) > 10 and - sum by (job) (rate(http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"}[5m])) > 0 + sum by (job) (rate(http_request_duration_seconds_bucket{job=~"thanos-query.*", handler="query"}[5m])) > 0 ) for: 10m labels: severity: critical -- alert: ThanosQuerierRangeLatencyHigh +- alert: ThanosQueryRangeLatencyHigh annotations: - message: Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ $value + message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries. expr: | ( - histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query_range"})) > 10 + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-query.*", handler="query_range"})) > 10 and - sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-querier.*", handler="query_range"}[5m])) > 0 + sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-query.*", handler="query_range"}[5m])) > 0 ) for: 10m labels: @@ -360,68 +360,68 @@ rules: ## Receive -[embedmd]:# (../tmp/thanos-receiver.rules.yaml yaml) +[embedmd]:# (../tmp/thanos-receive.rules.yaml yaml) ```yaml -name: thanos-receiver.rules +name: thanos-receive.rules rules: -- alert: ThanosReceiverHttpRequestErrorRateHigh +- alert: ThanosReceiveHttpRequestErrorRateHigh annotations: message: Thanos Receive {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests. expr: | ( - sum(rate(http_requests_total{code=~"5..", job=~"thanos-receiver.*", handler="receive"}[5m])) + sum(rate(http_requests_total{code=~"5..", job=~"thanos-receive.*", handler="receive"}[5m])) / - sum(rate(http_requests_total{job=~"thanos-receiver.*", handler="receive"}[5m])) + sum(rate(http_requests_total{job=~"thanos-receive.*", handler="receive"}[5m])) ) * 100 > 5 for: 5m labels: severity: critical -- alert: ThanosReceiverHttpRequestLatencyHigh +- alert: ThanosReceiveHttpRequestLatencyHigh annotations: message: Thanos Receive {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for requests. expr: | ( - histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-receiver.*", handler="receive"})) > 10 + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-receive.*", handler="receive"})) > 10 and - sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-receiver.*", handler="receive"}[5m])) > 0 + sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-receive.*", handler="receive"}[5m])) > 0 ) for: 10m labels: severity: critical -- alert: ThanosReceiverHighForwardRequestFailures +- alert: ThanosReceiveHighForwardRequestFailures annotations: message: Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize }}% of requests. expr: | ( - sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receiver.*"}[5m])) + sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receive.*"}[5m])) / - sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receiver.*"}[5m])) + sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receive.*"}[5m])) * 100 > 5 ) for: 5m labels: severity: critical -- alert: ThanosReceiverHighHashringFileRefreshFailures +- alert: ThanosReceiveHighHashringFileRefreshFailures annotations: message: Thanos Receive {{$labels.job}} is failing to refresh hashring file, {{ $value | humanize }} of attempts failed. expr: | ( - sum by (job) (rate(thanos_receive_hashrings_file_errors_total{job=~"thanos-receiver.*"}[5m])) + sum by (job) (rate(thanos_receive_hashrings_file_errors_total{job=~"thanos-receive.*"}[5m])) / - sum by (job) (rate(thanos_receive_hashrings_file_refreshes_total{job=~"thanos-receiver.*"}[5m])) + sum by (job) (rate(thanos_receive_hashrings_file_refreshes_total{job=~"thanos-receive.*"}[5m])) > 0 ) for: 15m labels: severity: warning -- alert: ThanosReceiverConfigReloadFailure +- alert: ThanosReceiveConfigReloadFailure annotations: message: Thanos Receive {{$labels.job}} has not been able to reload hashring configurations. - expr: avg(thanos_receive_config_last_reload_successful{job=~"thanos-receiver.*"}) + expr: avg(thanos_receive_config_last_reload_successful{job=~"thanos-receive.*"}) by (job) != 1 for: 5m labels: @@ -436,35 +436,35 @@ rules: ```yaml name: thanos-component-absent.rules rules: -- alert: ThanosCompactorIsDown +- alert: ThanosCompactIsDown annotations: - message: ThanosCompactor has disappeared from Prometheus target discovery. + message: ThanosCompact has disappeared from Prometheus target discovery. expr: | - absent(up{job=~"thanos-compactor.*"} == 1) + absent(up{job=~"thanos-compact.*"} == 1) for: 5m labels: severity: critical -- alert: ThanosQuerierIsDown +- alert: ThanosQueryIsDown annotations: - message: ThanosQuerier has disappeared from Prometheus target discovery. + message: ThanosQuery has disappeared from Prometheus target discovery. expr: | - absent(up{job=~"thanos-querier.*"} == 1) + absent(up{job=~"thanos-query.*"} == 1) for: 5m labels: severity: critical -- alert: ThanosReceiverIsDown +- alert: ThanosReceiveIsDown annotations: - message: ThanosReceiver has disappeared from Prometheus target discovery. + message: ThanosReceive has disappeared from Prometheus target discovery. expr: | - absent(up{job=~"thanos-receiver.*"} == 1) + absent(up{job=~"thanos-receive.*"} == 1) for: 5m labels: severity: critical -- alert: ThanosRulerIsDown +- alert: ThanosRuleIsDown annotations: - message: ThanosRuler has disappeared from Prometheus target discovery. + message: ThanosRule has disappeared from Prometheus target discovery. expr: | - absent(up{job=~"thanos-ruler.*"} == 1) + absent(up{job=~"thanos-rule.*"} == 1) for: 5m labels: severity: critical diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index 8db6674442..979231ae9e 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -1,211 +1,211 @@ groups: -- name: thanos-compactor.rules +- name: thanos-compact.rules rules: - - alert: ThanosCompactorMultipleCompactorsAreRunning + - alert: ThanosCompactMultipleRunning annotations: - message: You should never run more than one Thanos Compactor at once. You have - {{ $value }} - expr: sum(up{job=~"thanos-compactor.*"}) > 1 + message: No more than one Thanos Compact instance should be running at once. + There are {{ $value }} + expr: sum(up{job=~"thanos-compact.*"}) > 1 for: 5m labels: severity: warning - - alert: ThanosCompactorHalted + - alert: ThanosCompactHalted annotations: - message: Thanos Compactor {{$labels.job}} has failed to run and now is halted. - expr: thanos_compactor_halted{job=~"thanos-compactor.*"} == 1 + message: Thanos Compact {{$labels.job}} has failed to run and now is halted. + expr: thanos_compactor_halted{job=~"thanos-compact.*"} == 1 for: 5m labels: severity: warning - - alert: ThanosCompactorHighCompactionFailures + - alert: ThanosCompactHighCompactionFailures annotations: - message: Thanos Compactor {{$labels.job}} is failing to execute {{ $value | - humanize }}% of compactions. + message: Thanos Compact {{$labels.job}} is failing to execute {{ $value | humanize + }}% of compactions. expr: | ( - sum by (job) (rate(thanos_compact_group_compactions_failures_total{job=~"thanos-compactor.*"}[5m])) + sum by (job) (rate(thanos_compact_group_compactions_failures_total{job=~"thanos-compact.*"}[5m])) / - sum by (job) (rate(thanos_compact_group_compactions_total{job=~"thanos-compactor.*"}[5m])) + sum by (job) (rate(thanos_compact_group_compactions_total{job=~"thanos-compact.*"}[5m])) * 100 > 5 ) for: 15m labels: severity: warning - - alert: ThanosCompactorBucketHighOperationFailures + - alert: ThanosCompactBucketHighOperationFailures annotations: - message: Thanos Compactor {{$labels.job}} Bucket is failing to execute {{ $value + message: Thanos Compact {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations. expr: | ( - sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-compactor.*"}[5m])) + sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-compact.*"}[5m])) / - sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~"thanos-compactor.*"}[5m])) + sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~"thanos-compact.*"}[5m])) * 100 > 5 ) for: 15m labels: severity: warning - - alert: ThanosCompactorHasNotRun + - alert: ThanosCompactHasNotRun annotations: - message: Thanos Compactor {{$labels.job}} has not uploaded anything for 24 hours. - expr: (time() - max(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compactor.*"})) + message: Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours. + expr: (time() - max(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compact.*"})) / 60 / 60 > 24 labels: severity: warning -- name: thanos-querier.rules +- name: thanos-query.rules rules: - - alert: ThanosQuerierHttpRequestQueryErrorRateHigh + - alert: ThanosQueryHttpRequestQueryErrorRateHigh annotations: - message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests. expr: | ( - sum(rate(http_requests_total{code=~"5..", job=~"thanos-querier.*", handler="query"}[5m])) + sum(rate(http_requests_total{code=~"5..", job=~"thanos-query.*", handler="query"}[5m])) / - sum(rate(http_requests_total{job=~"thanos-querier.*", handler="query"}[5m])) + sum(rate(http_requests_total{job=~"thanos-query.*", handler="query"}[5m])) ) * 100 > 5 for: 5m labels: severity: critical - - alert: ThanosQuerierHttpRequestQueryRangeErrorRateHigh + - alert: ThanosQueryHttpRequestQueryRangeErrorRateHigh annotations: - message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests. expr: | ( - sum(rate(http_requests_total{code=~"5..", job=~"thanos-querier.*", handler="query_range"}[5m])) + sum(rate(http_requests_total{code=~"5..", job=~"thanos-query.*", handler="query_range"}[5m])) / - sum(rate(http_requests_total{job=~"thanos-querier.*", handler="query_range"}[5m])) + sum(rate(http_requests_total{job=~"thanos-query.*", handler="query_range"}[5m])) ) * 100 > 5 for: 5m labels: severity: critical - - alert: ThanosQuerierGrpcServerErrorRate + - alert: ThanosQueryGrpcServerErrorRate annotations: - message: Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests. expr: | ( - sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-query.*"}[5m])) / - sum by (job) (rate(grpc_server_started_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(grpc_server_started_total{job=~"thanos-query.*"}[5m])) * 100 > 5 ) for: 5m labels: severity: warning - - alert: ThanosQuerierGrpcClientErrorRate + - alert: ThanosQueryGrpcClientErrorRate annotations: - message: Thanos Querier {{$labels.job}} is failing to send {{ $value | humanize + message: Thanos Query {{$labels.job}} is failing to send {{ $value | humanize }}% of requests. expr: | ( - sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job=~"thanos-query.*"}[5m])) / - sum by (job) (rate(grpc_client_started_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(grpc_client_started_total{job=~"thanos-query.*"}[5m])) ) * 100 > 5 for: 5m labels: severity: warning - - alert: ThanosQuerierHighDNSFailures + - alert: ThanosQueryHighDNSFailures annotations: - message: Thanos Queriers {{$labels.job}} have {{ $value | humanize }}% of failing + message: Thanos Query {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for store endpoints. expr: | ( - sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-query.*"}[5m])) / - sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-query.*"}[5m])) ) * 100 > 1 for: 15m labels: severity: warning - - alert: ThanosQuerierInstantLatencyHigh + - alert: ThanosQueryInstantLatencyHigh annotations: - message: Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ - $value }} seconds for instant queries. + message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value + }} seconds for instant queries. expr: | ( - histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"})) > 10 + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-query.*", handler="query"})) > 10 and - sum by (job) (rate(http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"}[5m])) > 0 + sum by (job) (rate(http_request_duration_seconds_bucket{job=~"thanos-query.*", handler="query"}[5m])) > 0 ) for: 10m labels: severity: critical - - alert: ThanosQuerierRangeLatencyHigh + - alert: ThanosQueryRangeLatencyHigh annotations: - message: Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ - $value }} seconds for instant queries. + message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value + }} seconds for instant queries. expr: | ( - histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query_range"})) > 10 + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-query.*", handler="query_range"})) > 10 and - sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-querier.*", handler="query_range"}[5m])) > 0 + sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-query.*", handler="query_range"}[5m])) > 0 ) for: 10m labels: severity: critical -- name: thanos-receiver.rules +- name: thanos-receive.rules rules: - - alert: ThanosReceiverHttpRequestErrorRateHigh + - alert: ThanosReceiveHttpRequestErrorRateHigh annotations: message: Thanos Receive {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests. expr: | ( - sum(rate(http_requests_total{code=~"5..", job=~"thanos-receiver.*", handler="receive"}[5m])) + sum(rate(http_requests_total{code=~"5..", job=~"thanos-receive.*", handler="receive"}[5m])) / - sum(rate(http_requests_total{job=~"thanos-receiver.*", handler="receive"}[5m])) + sum(rate(http_requests_total{job=~"thanos-receive.*", handler="receive"}[5m])) ) * 100 > 5 for: 5m labels: severity: critical - - alert: ThanosReceiverHttpRequestLatencyHigh + - alert: ThanosReceiveHttpRequestLatencyHigh annotations: message: Thanos Receive {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for requests. expr: | ( - histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-receiver.*", handler="receive"})) > 10 + histogram_quantile(0.99, sum by (job, le) (http_request_duration_seconds_bucket{job=~"thanos-receive.*", handler="receive"})) > 10 and - sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-receiver.*", handler="receive"}[5m])) > 0 + sum by (job) (rate(http_request_duration_seconds_count{job=~"thanos-receive.*", handler="receive"}[5m])) > 0 ) for: 10m labels: severity: critical - - alert: ThanosReceiverHighForwardRequestFailures + - alert: ThanosReceiveHighForwardRequestFailures annotations: message: Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize }}% of requests. expr: | ( - sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receiver.*"}[5m])) + sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receive.*"}[5m])) / - sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receiver.*"}[5m])) + sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receive.*"}[5m])) * 100 > 5 ) for: 5m labels: severity: critical - - alert: ThanosReceiverHighHashringFileRefreshFailures + - alert: ThanosReceiveHighHashringFileRefreshFailures annotations: message: Thanos Receive {{$labels.job}} is failing to refresh hashring file, {{ $value | humanize }} of attempts failed. expr: | ( - sum by (job) (rate(thanos_receive_hashrings_file_errors_total{job=~"thanos-receiver.*"}[5m])) + sum by (job) (rate(thanos_receive_hashrings_file_errors_total{job=~"thanos-receive.*"}[5m])) / - sum by (job) (rate(thanos_receive_hashrings_file_refreshes_total{job=~"thanos-receiver.*"}[5m])) + sum by (job) (rate(thanos_receive_hashrings_file_refreshes_total{job=~"thanos-receive.*"}[5m])) > 0 ) for: 15m labels: severity: warning - - alert: ThanosReceiverConfigReloadFailure + - alert: ThanosReceiveConfigReloadFailure annotations: message: Thanos Receive {{$labels.job}} has not been able to reload hashring configurations. - expr: avg(thanos_receive_config_last_reload_successful{job=~"thanos-receiver.*"}) + expr: avg(thanos_receive_config_last_reload_successful{job=~"thanos-receive.*"}) by (job) != 1 for: 5m labels: @@ -284,106 +284,106 @@ groups: for: 10m labels: severity: warning -- name: thanos-ruler.rules +- name: thanos-rule.rules rules: - - alert: ThanosRulerQueueIsDroppingAlerts + - alert: ThanosRuleQueueIsDroppingAlerts annotations: - message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to queue alerts. + message: Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to queue alerts. expr: | - sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=~"thanos-ruler.*"}[5m])) > 0 + sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=~"thanos-rule.*"}[5m])) > 0 for: 5m labels: severity: critical - - alert: ThanosRulerSenderIsFailingAlerts + - alert: ThanosRuleSenderIsFailingAlerts annotations: - message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to send alerts + message: Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to send alerts to alertmanager. expr: | - sum by (job) (rate(thanos_alert_sender_alerts_dropped_total{job=~"thanos-ruler.*"}[5m])) > 0 + sum by (job) (rate(thanos_alert_sender_alerts_dropped_total{job=~"thanos-rule.*"}[5m])) > 0 for: 5m labels: severity: critical - - alert: ThanosRulerHighRuleEvaluationFailures + - alert: ThanosRuleHighRuleEvaluationFailures annotations: - message: Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to evaluate + message: Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to evaluate rules. expr: | ( - sum by (job) (rate(prometheus_rule_evaluation_failures_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(prometheus_rule_evaluation_failures_total{job=~"thanos-rule.*"}[5m])) / - sum by (job) (rate(prometheus_rule_evaluations_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(prometheus_rule_evaluations_total{job=~"thanos-rule.*"}[5m])) * 100 > 5 ) for: 5m labels: severity: warning - - alert: ThanosRulerHighRuleEvaluationWarnings + - alert: ThanosRuleHighRuleEvaluationWarnings annotations: - message: Thanos Ruler {{$labels.job}} {{$labels.pod}} has high number of evaluation + message: Thanos Rule {{$labels.job}} {{$labels.pod}} has high number of evaluation warnings. expr: | - sum by (job) (rate(thanos_rule_evaluation_with_warnings_total{job=~"thanos-ruler.*"}[5m])) > 0 + sum by (job) (rate(thanos_rule_evaluation_with_warnings_total{job=~"thanos-rule.*"}[5m])) > 0 for: 15m labels: severity: warning - - alert: ThanosRulerRuleEvaluationLatencyHigh + - alert: ThanosRuleRuleEvaluationLatencyHigh annotations: - message: Thanos Ruler {{$labels.job}}/{{$labels.pod}} has higher evaluation - latency than interval for {{$labels.rule_group}}. + message: Thanos Rule {{$labels.job}}/{{$labels.pod}} has higher evaluation latency + than interval for {{$labels.rule_group}}. expr: | ( - sum by (job, pod, rule_group) (prometheus_rule_group_last_duration_seconds{job=~"thanos-ruler.*"}) + sum by (job, pod, rule_group) (prometheus_rule_group_last_duration_seconds{job=~"thanos-rule.*"}) > - sum by (job, pod, rule_group) (prometheus_rule_group_interval_seconds{job=~"thanos-ruler.*"}) + sum by (job, pod, rule_group) (prometheus_rule_group_interval_seconds{job=~"thanos-rule.*"}) ) for: 5m labels: severity: warning - - alert: ThanosRulerGrpcErrorRate + - alert: ThanosRuleGrpcErrorRate annotations: - message: Thanos Ruler {{$labels.job}} is failing to handle {{ $value | humanize + message: Thanos Rule {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests. expr: | ( - sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-rule.*"}[5m])) / - sum by (job) (rate(grpc_server_started_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(grpc_server_started_total{job=~"thanos-rule.*"}[5m])) * 100 > 5 ) for: 5m labels: severity: warning - - alert: ThanosRulerConfigReloadFailure + - alert: ThanosRuleConfigReloadFailure annotations: - message: Thanos Ruler {{$labels.job}} has not been able to reload its configuration. - expr: avg(thanos_rule_config_last_reload_successful{job=~"thanos-ruler.*"}) by + message: Thanos Rule {{$labels.job}} has not been able to reload its configuration. + expr: avg(thanos_rule_config_last_reload_successful{job=~"thanos-rule.*"}) by (job) != 1 for: 5m labels: severity: warning - - alert: ThanosRulerQueryHighDNSFailures + - alert: ThanosRuleQueryHighDNSFailures annotations: - message: Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing + message: Thanos Rule {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for query endpoints. expr: | ( - sum by (job) (rate(thanos_ruler_query_apis_dns_failures_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(thanos_ruler_query_apis_dns_failures_total{job=~"thanos-rule.*"}[5m])) / - sum by (job) (rate(thanos_ruler_query_apis_dns_lookups_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(thanos_ruler_query_apis_dns_lookups_total{job=~"thanos-rule.*"}[5m])) * 100 > 1 ) for: 15m labels: severity: warning - - alert: ThanosRulerAlertmanagerHighDNSFailures + - alert: ThanosRuleAlertmanagerHighDNSFailures annotations: - message: Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing + message: Thanos Rule {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for Alertmanager endpoints. expr: | ( - sum by (job) (rate(thanos_ruler_alertmanagers_dns_failures_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(thanos_ruler_alertmanagers_dns_failures_total{job=~"thanos-rule.*"}[5m])) / - sum by (job) (rate(thanos_ruler_alertmanagers_dns_lookups_total{job=~"thanos-ruler.*"}[5m])) + sum by (job) (rate(thanos_ruler_alertmanagers_dns_lookups_total{job=~"thanos-rule.*"}[5m])) * 100 > 1 ) for: 15m @@ -391,35 +391,35 @@ groups: severity: warning - name: thanos-component-absent.rules rules: - - alert: ThanosCompactorIsDown + - alert: ThanosCompactIsDown annotations: - message: ThanosCompactor has disappeared from Prometheus target discovery. + message: ThanosCompact has disappeared from Prometheus target discovery. expr: | - absent(up{job=~"thanos-compactor.*"} == 1) + absent(up{job=~"thanos-compact.*"} == 1) for: 5m labels: severity: critical - - alert: ThanosQuerierIsDown + - alert: ThanosQueryIsDown annotations: - message: ThanosQuerier has disappeared from Prometheus target discovery. + message: ThanosQuery has disappeared from Prometheus target discovery. expr: | - absent(up{job=~"thanos-querier.*"} == 1) + absent(up{job=~"thanos-query.*"} == 1) for: 5m labels: severity: critical - - alert: ThanosReceiverIsDown + - alert: ThanosReceiveIsDown annotations: - message: ThanosReceiver has disappeared from Prometheus target discovery. + message: ThanosReceive has disappeared from Prometheus target discovery. expr: | - absent(up{job=~"thanos-receiver.*"} == 1) + absent(up{job=~"thanos-receive.*"} == 1) for: 5m labels: severity: critical - - alert: ThanosRulerIsDown + - alert: ThanosRuleIsDown annotations: - message: ThanosRuler has disappeared from Prometheus target discovery. + message: ThanosRule has disappeared from Prometheus target discovery. expr: | - absent(up{job=~"thanos-ruler.*"} == 1) + absent(up{job=~"thanos-rule.*"} == 1) for: 5m labels: severity: critical diff --git a/examples/alerts/rules.yaml b/examples/alerts/rules.yaml index 82190e63c6..9049262121 100644 --- a/examples/alerts/rules.yaml +++ b/examples/alerts/rules.yaml @@ -1,90 +1,90 @@ groups: -- name: thanos-querier.rules +- name: thanos-query.rules rules: - expr: | ( - sum(rate(grpc_client_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-querier.*", grpc_type="unary"}[5m])) + sum(rate(grpc_client_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-query.*", grpc_type="unary"}[5m])) / - sum(rate(grpc_client_started_total{job=~"thanos-querier.*", grpc_type="unary"}[5m])) + sum(rate(grpc_client_started_total{job=~"thanos-query.*", grpc_type="unary"}[5m])) ) labels: {} record: :grpc_client_failures_per_unary:sum_rate - expr: | ( - sum(rate(grpc_client_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-querier.*", grpc_type="server_stream"}[5m])) + sum(rate(grpc_client_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-query.*", grpc_type="server_stream"}[5m])) / - sum(rate(grpc_client_started_total{job=~"thanos-querier.*", grpc_type="server_stream"}[5m])) + sum(rate(grpc_client_started_total{job=~"thanos-query.*", grpc_type="server_stream"}[5m])) ) labels: {} record: :grpc_client_failures_per_stream:sum_rate - expr: | ( - sum(rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-querier.*"}[5m])) + sum(rate(thanos_querier_store_apis_dns_failures_total{job=~"thanos-query.*"}[5m])) / - sum(rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-querier.*"}[5m])) + sum(rate(thanos_querier_store_apis_dns_lookups_total{job=~"thanos-query.*"}[5m])) ) labels: {} record: :thanos_querier_store_apis_dns_failures_per_lookup:sum_rate - expr: | histogram_quantile(0.99, - sum(rate(http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query"}[5m])) by (le) + sum(rate(http_request_duration_seconds_bucket{job=~"thanos-query.*", handler="query"}[5m])) by (le) ) labels: quantile: "0.99" record: :query_duration_seconds:histogram_quantile - expr: | histogram_quantile(0.99, - sum(rate(http_request_duration_seconds_bucket{job=~"thanos-querier.*", handler="query_range"}[5m])) by (le) + sum(rate(http_request_duration_seconds_bucket{job=~"thanos-query.*", handler="query_range"}[5m])) by (le) ) labels: quantile: "0.99" record: :api_range_query_duration_seconds:histogram_quantile -- name: thanos-receiver.rules +- name: thanos-receive.rules rules: - expr: | sum( - rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-receiver.*", grpc_type="unary"}[5m]) + rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-receive.*", grpc_type="unary"}[5m]) / - rate(grpc_server_started_total{job=~"thanos-receiver.*", grpc_type="unary"}[5m]) + rate(grpc_server_started_total{job=~"thanos-receive.*", grpc_type="unary"}[5m]) ) labels: {} record: :grpc_server_failures_per_unary:sum_rate - expr: | sum( - rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-receiver.*", grpc_type="server_stream"}[5m]) + rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~"thanos-receive.*", grpc_type="server_stream"}[5m]) / - rate(grpc_server_started_total{job=~"thanos-receiver.*", grpc_type="server_stream"}[5m]) + rate(grpc_server_started_total{job=~"thanos-receive.*", grpc_type="server_stream"}[5m]) ) labels: {} record: :grpc_server_failures_per_stream:sum_rate - expr: | sum( - rate(http_requests_total{handler="receive", job=~"thanos-receiver.*", code!~"5.."}[5m]) + rate(http_requests_total{handler="receive", job=~"thanos-receive.*", code!~"5.."}[5m]) / - rate(http_requests_total{handler="receive", job=~"thanos-receiver.*"}[5m]) + rate(http_requests_total{handler="receive", job=~"thanos-receive.*"}[5m]) ) labels: {} record: :http_failure_per_request:sum_rate - expr: | histogram_quantile(0.99, - sum(rate(http_request_duration_seconds_bucket{handler="receive", job=~"thanos-receiver.*"}[5m])) by (le) + sum(rate(http_request_duration_seconds_bucket{handler="receive", job=~"thanos-receive.*"}[5m])) by (le) ) labels: quantile: "0.99" record: :http_request_duration_seconds:histogram_quantile - expr: | ( - sum(rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receiver.*"}[5m])) + sum(rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receive.*"}[5m])) / - sum(rate(thanos_receive_forward_requests_total{job=~"thanos-receiver.*"}[5m])) + sum(rate(thanos_receive_forward_requests_total{job=~"thanos-receive.*"}[5m])) ) labels: {} record: :thanos_receive_forward_failure_per_requests:sum_rate - expr: | ( - sum(rate(thanos_receive_hashrings_file_errors_total{job=~"thanos-receiver.*"}[5m])) + sum(rate(thanos_receive_hashrings_file_errors_total{job=~"thanos-receive.*"}[5m])) / - sum(rate(thanos_receive_hashrings_file_refreshes_total{job=~"thanos-receiver.*"}[5m])) + sum(rate(thanos_receive_hashrings_file_refreshes_total{job=~"thanos-receive.*"}[5m])) ) labels: {} record: :thanos_receive_hashring_file_failure_per_refresh:sum_rate diff --git a/examples/dashboards/compactor.json b/examples/dashboards/compact.json similarity index 99% rename from examples/dashboards/compactor.json rename to examples/dashboards/compact.json index ba36ef58db..44fe9a625d 100644 --- a/examples/dashboards/compactor.json +++ b/examples/dashboards/compact.json @@ -1451,7 +1451,7 @@ "useTags": false }, { - "allValue": "thanos-compactor.*", + "allValue": "thanos-compact.*", "current": { "text": "all", "value": "$__all" @@ -1463,7 +1463,7 @@ "multi": false, "name": "job", "options": [ ], - "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-compactor.*\"}, job)", + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-compact.*\"}, job)", "refresh": 1, "regex": "", "sort": 2, @@ -1486,7 +1486,7 @@ "multi": false, "name": "pod", "options": [ ], - "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-compactor.*\"}, pod)", + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-compact.*\"}, pod)", "refresh": 1, "regex": "", "sort": 2, @@ -1543,7 +1543,7 @@ ] }, "timezone": "", - "title": "Thanos / Compactor", - "uid": "8378cc7f803b0bfae5809a101991ed76", + "title": "Thanos / Compact", + "uid": "651943d05a8123e32867b4673963f42b", "version": 0 } diff --git a/examples/dashboards/overview.json b/examples/dashboards/overview.json index ebd45038ca..79a383fd88 100644 --- a/examples/dashboards/overview.json +++ b/examples/dashboards/overview.json @@ -43,10 +43,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Querier", + "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, - "title": "Thanos / Querier", + "title": "Thanos / Query", "type": "dashboard" } ], @@ -62,7 +62,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(label_replace(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", + "expr": "sum(label_replace(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-query.*\",handler=\"query\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{status_code}}", @@ -130,10 +130,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Querier", + "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, - "title": "Thanos / Querier", + "title": "Thanos / Query", "type": "dashboard" } ], @@ -149,7 +149,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query\"}[$interval]))", + "expr": "sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-query.*\",handler=\"query\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-query.*\",handler=\"query\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", @@ -215,10 +215,10 @@ "linewidth": 1, "links": [ { - "dashboard": "Thanos / Querier", + "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, - "title": "Thanos / Querier", + "title": "Thanos / Query", "type": "dashboard" } ], @@ -234,7 +234,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query\"}[$interval])) by (job, le))", + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"thanos-query.*\",handler=\"query\"}[$interval])) by (job, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", @@ -337,10 +337,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Querier", + "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, - "title": "Thanos / Querier", + "title": "Thanos / Query", "type": "dashboard" } ], @@ -356,7 +356,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(label_replace(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query_range\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", + "expr": "sum(label_replace(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-query.*\",handler=\"query_range\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{status_code}}", @@ -424,10 +424,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Querier", + "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, - "title": "Thanos / Querier", + "title": "Thanos / Query", "type": "dashboard" } ], @@ -443,7 +443,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query_range\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query_range\"}[$interval]))", + "expr": "sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-query.*\",handler=\"query_range\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{namespace=\"$namespace\",job=~\"thanos-query.*\",handler=\"query_range\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", @@ -509,10 +509,10 @@ "linewidth": 1, "links": [ { - "dashboard": "Thanos / Querier", + "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, - "title": "Thanos / Querier", + "title": "Thanos / Query", "type": "dashboard" } ], @@ -528,7 +528,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"thanos-querier.*\",handler=\"query_range\"}[$interval])) by (job, le))", + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{namespace=\"$namespace\",job=~\"thanos-query.*\",handler=\"query_range\"}[$interval])) by (job, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", @@ -1241,10 +1241,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Receiver", + "dashboard": "Thanos / Receive", "includeVars": true, "keepTime": true, - "title": "Thanos / Receiver", + "title": "Thanos / Receive", "type": "dashboard" } ], @@ -1260,7 +1260,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(label_replace(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receiver.*\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", + "expr": "sum(label_replace(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receive.*\"}[$interval]),\"status_code\", \"${1}xx\", \"code\", \"([0-9])..\")) by (job, status_code)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{status_code}}", @@ -1328,10 +1328,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Receiver", + "dashboard": "Thanos / Receive", "includeVars": true, "keepTime": true, - "title": "Thanos / Receiver", + "title": "Thanos / Receive", "type": "dashboard" } ], @@ -1347,7 +1347,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receiver.*\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receiver.*\"}[$interval]))", + "expr": "sum(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receive.*\",code=~\"5..\"}[$interval])) / sum(rate(http_requests_total{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receive.*\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", @@ -1413,10 +1413,10 @@ "linewidth": 1, "links": [ { - "dashboard": "Thanos / Receiver", + "dashboard": "Thanos / Receive", "includeVars": true, "keepTime": true, - "title": "Thanos / Receiver", + "title": "Thanos / Receive", "type": "dashboard" } ], @@ -1432,7 +1432,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receiver.*\"}[$interval])) by (job, le))", + "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{handler=\"receive\",namespace=\"$namespace\",job=~\"thanos-receive.*\"}[$interval])) by (job, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", @@ -1527,10 +1527,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Ruler", + "dashboard": "Thanos / Rule", "includeVars": true, "keepTime": true, - "title": "Thanos / Ruler", + "title": "Thanos / Rule", "type": "dashboard" } ], @@ -1546,7 +1546,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}[$interval])) by (job, alertmanager)", + "expr": "sum(rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\",job=~\"thanos-rule.*\"}[$interval])) by (job, alertmanager)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{alertmanager}}", @@ -1614,10 +1614,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Ruler", + "dashboard": "Thanos / Rule", "includeVars": true, "keepTime": true, - "title": "Thanos / Ruler", + "title": "Thanos / Rule", "type": "dashboard" } ], @@ -1633,7 +1633,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(thanos_alert_sender_errors_total{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}[$interval])) / sum(rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}[$interval]))", + "expr": "sum(rate(thanos_alert_sender_errors_total{namespace=\"$namespace\",job=~\"thanos-rule.*\"}[$interval])) / sum(rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\",job=~\"thanos-rule.*\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", @@ -1699,10 +1699,10 @@ "linewidth": 1, "links": [ { - "dashboard": "Thanos / Ruler", + "dashboard": "Thanos / Rule", "includeVars": true, "keepTime": true, - "title": "Thanos / Ruler", + "title": "Thanos / Rule", "type": "dashboard" } ], @@ -1718,7 +1718,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(thanos_alert_sender_latency_seconds_bucket{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}[$interval])) by (job, le))", + "expr": "histogram_quantile(0.99, sum(rate(thanos_alert_sender_latency_seconds_bucket{namespace=\"$namespace\",job=~\"thanos-rule.*\"}[$interval])) by (job, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", @@ -1813,10 +1813,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Compactor", + "dashboard": "Thanos / Compact", "includeVars": true, "keepTime": true, - "title": "Thanos / Compactor", + "title": "Thanos / Compact", "type": "dashboard" } ], @@ -1832,7 +1832,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(thanos_compact_group_compactions_total{namespace=\"$namespace\",job=~\"thanos-compactor.*\"}[$interval])) by (job)", + "expr": "sum(rate(thanos_compact_group_compactions_total{namespace=\"$namespace\",job=~\"thanos-compact.*\"}[$interval])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "compaction {{job}}", @@ -1900,10 +1900,10 @@ "linewidth": 0, "links": [ { - "dashboard": "Thanos / Compactor", + "dashboard": "Thanos / Compact", "includeVars": true, "keepTime": true, - "title": "Thanos / Compactor", + "title": "Thanos / Compact", "type": "dashboard" } ], @@ -1919,7 +1919,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(thanos_compact_group_compactions_failures_total{namespace=\"$namespace\",job=~\"thanos-compactor.*\"}[$interval])) / sum(rate(thanos_compact_group_compactions_total{namespace=\"$namespace\",job=~\"thanos-compactor.*\"}[$interval]))", + "expr": "sum(rate(thanos_compact_group_compactions_failures_total{namespace=\"$namespace\",job=~\"thanos-compact.*\"}[$interval])) / sum(rate(thanos_compact_group_compactions_total{namespace=\"$namespace\",job=~\"thanos-compact.*\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", @@ -1968,7 +1968,7 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Compactor", + "title": "Compact", "titleSize": "h6" } ], diff --git a/examples/dashboards/querier.json b/examples/dashboards/query.json similarity index 99% rename from examples/dashboards/querier.json rename to examples/dashboards/query.json index f69e569ab8..9f1bfa7ef2 100644 --- a/examples/dashboards/querier.json +++ b/examples/dashboards/query.json @@ -2383,7 +2383,7 @@ "useTags": false }, { - "allValue": "thanos-querier.*", + "allValue": "thanos-query.*", "current": { "text": "all", "value": "$__all" @@ -2395,7 +2395,7 @@ "multi": false, "name": "job", "options": [ ], - "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-querier.*\"}, job)", + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-query.*\"}, job)", "refresh": 1, "regex": "", "sort": 2, @@ -2418,7 +2418,7 @@ "multi": false, "name": "pod", "options": [ ], - "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-querier.*\"}, pod)", + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-query.*\"}, pod)", "refresh": 1, "regex": "", "sort": 2, @@ -2475,7 +2475,7 @@ ] }, "timezone": "", - "title": "Thanos / Querier", - "uid": "98fde97ddeaf2981041745f1f2ba68c2", + "title": "Thanos / Query", + "uid": "af36c91291a603f1d9fbdabdd127ac4a", "version": 0 } diff --git a/examples/dashboards/receiver.json b/examples/dashboards/receive.json similarity index 99% rename from examples/dashboards/receiver.json rename to examples/dashboards/receive.json index 3abba13206..1c3081d0cf 100644 --- a/examples/dashboards/receiver.json +++ b/examples/dashboards/receive.json @@ -2238,7 +2238,7 @@ "useTags": false }, { - "allValue": "thanos-receiver.*", + "allValue": "thanos-receive.*", "current": { "text": "all", "value": "$__all" @@ -2250,7 +2250,7 @@ "multi": false, "name": "job", "options": [ ], - "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-receiver.*\"}, job)", + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-receive.*\"}, job)", "refresh": 1, "regex": "", "sort": 2, @@ -2273,7 +2273,7 @@ "multi": false, "name": "pod", "options": [ ], - "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-receiver.*\"}, pod)", + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-receive.*\"}, pod)", "refresh": 1, "regex": "", "sort": 2, @@ -2330,7 +2330,7 @@ ] }, "timezone": "", - "title": "Thanos / Receiver", - "uid": "b5958da86b143e45752506d3c09c4f92", + "title": "Thanos / Receive", + "uid": "916a852b00ccc5ed81056644718fa4fb", "version": 0 } diff --git a/examples/dashboards/ruler.json b/examples/dashboards/rule.json similarity index 99% rename from examples/dashboards/ruler.json rename to examples/dashboards/rule.json index 6dc5f85a25..897b168dc2 100644 --- a/examples/dashboards/ruler.json +++ b/examples/dashboards/rule.json @@ -1916,7 +1916,7 @@ "useTags": false }, { - "allValue": "thanos-ruler.*", + "allValue": "thanos-rule.*", "current": { "text": "all", "value": "$__all" @@ -1928,7 +1928,7 @@ "multi": false, "name": "job", "options": [ ], - "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-ruler.*\"}, job)", + "query": "label_values(up{namespace=\"$namespace\",job=~\"thanos-rule.*\"}, job)", "refresh": 1, "regex": "", "sort": 2, @@ -1951,7 +1951,7 @@ "multi": false, "name": "pod", "options": [ ], - "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-ruler.*\"}, pod)", + "query": "label_values(kube_pod_info{namespace=\"$namespace\",created_by_name=~\"thanos-rule.*\"}, pod)", "refresh": 1, "regex": "", "sort": 2, @@ -2008,7 +2008,7 @@ ] }, "timezone": "", - "title": "Thanos / Ruler", - "uid": "ecc62fafbb8ae0213cb9188ec4f3b553", + "title": "Thanos / Rule", + "uid": "35da848f5f92b2dc612e0c3a0577b8a1", "version": 0 } diff --git a/mixin/thanos/README.md b/mixin/thanos/README.md index f53e11c509..946e7a916d 100644 --- a/mixin/thanos/README.md +++ b/mixin/thanos/README.md @@ -59,30 +59,30 @@ This project is intended to be used as a library. You can extend and customize d [embedmd]:# (defaults.libsonnet) ```libsonnet { - querier+:: { - jobPrefix: 'thanos-querier', + query+:: { + jobPrefix: 'thanos-query', selector: 'job=~"%s.*"' % self.jobPrefix, - title: '%(prefix)sQuerier' % $.dashboard.prefix, + title: '%(prefix)sQuery' % $.dashboard.prefix, }, store+:: { jobPrefix: 'thanos-store', selector: 'job=~"%s.*"' % self.jobPrefix, title: '%(prefix)sStore' % $.dashboard.prefix, }, - receiver+:: { - jobPrefix: 'thanos-receiver', + receive+:: { + jobPrefix: 'thanos-receive', selector: 'job=~"%s.*"' % self.jobPrefix, - title: '%(prefix)sReceiver' % $.dashboard.prefix, + title: '%(prefix)sReceive' % $.dashboard.prefix, }, - ruler+:: { - jobPrefix: 'thanos-ruler', + rule+:: { + jobPrefix: 'thanos-rule', selector: 'job=~"%s.*"' % self.jobPrefix, - title: '%(prefix)sRuler' % $.dashboard.prefix, + title: '%(prefix)sRule' % $.dashboard.prefix, }, - compactor+:: { - jobPrefix: 'thanos-compactor', + compact+:: { + jobPrefix: 'thanos-compact', selector: 'job=~"%s.*"' % self.jobPrefix, - title: '%(prefix)sCompactor' % $.dashboard.prefix, + title: '%(prefix)sCompact' % $.dashboard.prefix, }, sidecar+:: { jobPrefix: 'thanos-sidecar', diff --git a/mixin/thanos/alerts/absent.libsonnet b/mixin/thanos/alerts/absent.libsonnet index a134fc8f60..4c7bf6bdff 100644 --- a/mixin/thanos/alerts/absent.libsonnet +++ b/mixin/thanos/alerts/absent.libsonnet @@ -3,11 +3,11 @@ // We build alerts for the presence of all these jobs. jobs:: { - ThanosQuerier: thanos.querier.selector, + ThanosQuery: thanos.query.selector, ThanosStore: thanos.store.selector, - ThanosReceiver: thanos.receiver.selector, - ThanosRuler: thanos.ruler.selector, - ThanosCompactor: thanos.compactor.selector, + ThanosReceive: thanos.receive.selector, + ThanosRule: thanos.rule.selector, + ThanosCompact: thanos.compact.selector, ThanosSidecar: thanos.sidecar.selector, }, diff --git a/mixin/thanos/alerts/alerts.libsonnet b/mixin/thanos/alerts/alerts.libsonnet index e3fa004090..fa0a9d3d36 100644 --- a/mixin/thanos/alerts/alerts.libsonnet +++ b/mixin/thanos/alerts/alerts.libsonnet @@ -1,7 +1,7 @@ -(import 'compactor.libsonnet') + -(import 'querier.libsonnet') + -(import 'receiver.libsonnet') + +(import 'compact.libsonnet') + +(import 'query.libsonnet') + +(import 'receive.libsonnet') + (import 'sidecar.libsonnet') + (import 'store.libsonnet') + -(import 'ruler.libsonnet') + +(import 'rule.libsonnet') + (import 'absent.libsonnet') diff --git a/mixin/thanos/alerts/compactor.libsonnet b/mixin/thanos/alerts/compact.libsonnet similarity index 60% rename from mixin/thanos/alerts/compactor.libsonnet rename to mixin/thanos/alerts/compact.libsonnet index 3baab8b7d3..04c1115f2d 100644 --- a/mixin/thanos/alerts/compactor.libsonnet +++ b/mixin/thanos/alerts/compact.libsonnet @@ -1,40 +1,40 @@ { local thanos = self, - compactor+:: { - jobPrefix: error 'must provide job prefix for Thanos Compactor alerts', - selector: error 'must provide selector for Thanos Compactor alerts', + compact+:: { + jobPrefix: error 'must provide job prefix for Thanos Compact alerts', + selector: error 'must provide selector for Thanos Compact alerts', }, prometheusAlerts+:: { groups+: [ { - name: 'thanos-compactor.rules', + name: 'thanos-compact.rules', rules: [ { - alert: 'ThanosCompactorMultipleCompactorsAreRunning', + alert: 'ThanosCompactMultipleRunning', annotations: { - message: 'You should never run more than one Thanos Compactor at once. You have {{ $value }}', + message: 'No more than one Thanos Compact instance should be running at once. There are {{ $value }}', }, - expr: 'sum(up{%(selector)s}) > 1' % thanos.compactor, + expr: 'sum(up{%(selector)s}) > 1' % thanos.compact, 'for': '5m', labels: { severity: 'warning', }, }, { - alert: 'ThanosCompactorHalted', + alert: 'ThanosCompactHalted', annotations: { - message: 'Thanos Compactor {{$labels.job}} has failed to run and now is halted.', + message: 'Thanos Compact {{$labels.job}} has failed to run and now is halted.', }, - expr: 'thanos_compactor_halted{%(selector)s} == 1' % thanos.compactor, + expr: 'thanos_compactor_halted{%(selector)s} == 1' % thanos.compact, 'for': '5m', labels: { severity: 'warning', }, }, { - alert: 'ThanosCompactorHighCompactionFailures', + alert: 'ThanosCompactHighCompactionFailures', annotations: { - message: 'Thanos Compactor {{$labels.job}} is failing to execute {{ $value | humanize }}% of compactions.', + message: 'Thanos Compact {{$labels.job}} is failing to execute {{ $value | humanize }}% of compactions.', }, expr: ||| ( @@ -43,16 +43,16 @@ sum by (job) (rate(thanos_compact_group_compactions_total{%(selector)s}[5m])) * 100 > 5 ) - ||| % thanos.compactor, + ||| % thanos.compact, 'for': '15m', labels: { severity: 'warning', }, }, { - alert: 'ThanosCompactorBucketHighOperationFailures', + alert: 'ThanosCompactBucketHighOperationFailures', annotations: { - message: 'Thanos Compactor {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations.', + message: 'Thanos Compact {{$labels.job}} Bucket is failing to execute {{ $value | humanize }}% of operations.', }, expr: ||| ( @@ -61,18 +61,18 @@ sum by (job) (rate(thanos_objstore_bucket_operations_total{%(selector)s}[5m])) * 100 > 5 ) - ||| % thanos.compactor, + ||| % thanos.compact, 'for': '15m', labels: { severity: 'warning', }, }, { - alert: 'ThanosCompactorHasNotRun', + alert: 'ThanosCompactHasNotRun', annotations: { - message: 'Thanos Compactor {{$labels.job}} has not uploaded anything for 24 hours.', + message: 'Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours.', }, - expr: '(time() - max(thanos_objstore_bucket_last_successful_upload_time{%(selector)s})) / 60 / 60 > 24' % thanos.compactor, + expr: '(time() - max(thanos_objstore_bucket_last_successful_upload_time{%(selector)s})) / 60 / 60 > 24' % thanos.compact, labels: { severity: 'warning', }, diff --git a/mixin/thanos/alerts/querier.libsonnet b/mixin/thanos/alerts/query.libsonnet similarity index 68% rename from mixin/thanos/alerts/querier.libsonnet rename to mixin/thanos/alerts/query.libsonnet index ca3ae2a218..8b8644f89f 100644 --- a/mixin/thanos/alerts/querier.libsonnet +++ b/mixin/thanos/alerts/query.libsonnet @@ -1,18 +1,18 @@ { local thanos = self, - querier+:: { + query+:: { jobPrefix: error 'must provide job prefix for Thanos Query alerts', selector: error 'must provide selector for Thanos Query alerts', }, prometheusAlerts+:: { groups+: [ { - name: 'thanos-querier.rules', + name: 'thanos-query.rules', rules: [ { - alert: 'ThanosQuerierHttpRequestQueryErrorRateHigh', + alert: 'ThanosQueryHttpRequestQueryErrorRateHigh', annotations: { - message: 'Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests.', + message: 'Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests.', }, expr: ||| ( @@ -20,16 +20,16 @@ / sum(rate(http_requests_total{%(selector)s, handler="query"}[5m])) ) * 100 > 5 - ||| % thanos.querier, + ||| % thanos.query, 'for': '5m', labels: { severity: 'critical', }, }, { - alert: 'ThanosQuerierHttpRequestQueryRangeErrorRateHigh', + alert: 'ThanosQueryHttpRequestQueryRangeErrorRateHigh', annotations: { - message: 'Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests.', + message: 'Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests.', }, expr: ||| ( @@ -37,16 +37,16 @@ / sum(rate(http_requests_total{%(selector)s, handler="query_range"}[5m])) ) * 100 > 5 - ||| % thanos.querier, + ||| % thanos.query, 'for': '5m', labels: { severity: 'critical', }, }, { - alert: 'ThanosQuerierGrpcServerErrorRate', + alert: 'ThanosQueryGrpcServerErrorRate', annotations: { - message: 'Thanos Querier {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', + message: 'Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', }, expr: ||| ( @@ -55,16 +55,16 @@ sum by (job) (rate(grpc_server_started_total{%(selector)s}[5m])) * 100 > 5 ) - ||| % thanos.querier, + ||| % thanos.query, 'for': '5m', labels: { severity: 'warning', }, }, { - alert: 'ThanosQuerierGrpcClientErrorRate', + alert: 'ThanosQueryGrpcClientErrorRate', annotations: { - message: 'Thanos Querier {{$labels.job}} is failing to send {{ $value | humanize }}% of requests.', + message: 'Thanos Query {{$labels.job}} is failing to send {{ $value | humanize }}% of requests.', }, expr: ||| ( @@ -72,16 +72,16 @@ / sum by (job) (rate(grpc_client_started_total{%(selector)s}[5m])) ) * 100 > 5 - ||| % thanos.querier, + ||| % thanos.query, 'for': '5m', labels: { severity: 'warning', }, }, { - alert: 'ThanosQuerierHighDNSFailures', + alert: 'ThanosQueryHighDNSFailures', annotations: { - message: 'Thanos Queriers {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for store endpoints.', + message: 'Thanos Query {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for store endpoints.', }, expr: ||| ( @@ -89,16 +89,16 @@ / sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{%(selector)s}[5m])) ) * 100 > 1 - ||| % thanos.querier, + ||| % thanos.query, 'for': '15m', labels: { severity: 'warning', }, }, { - alert: 'ThanosQuerierInstantLatencyHigh', + alert: 'ThanosQueryInstantLatencyHigh', annotations: { - message: 'Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', + message: 'Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', }, expr: ||| ( @@ -106,16 +106,16 @@ and sum by (job) (rate(http_request_duration_seconds_bucket{%(selector)s, handler="query"}[5m])) > 0 ) - ||| % thanos.querier, + ||| % thanos.query, 'for': '10m', labels: { severity: 'critical', }, }, { - alert: 'ThanosQuerierRangeLatencyHigh', + alert: 'ThanosQueryRangeLatencyHigh', annotations: { - message: 'Thanos Querier {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', + message: 'Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.', }, expr: ||| ( @@ -123,7 +123,7 @@ and sum by (job) (rate(http_request_duration_seconds_count{%(selector)s, handler="query_range"}[5m])) > 0 ) - ||| % thanos.querier, + ||| % thanos.query, 'for': '10m', labels: { severity: 'critical', diff --git a/mixin/thanos/alerts/receiver.libsonnet b/mixin/thanos/alerts/receive.libsonnet similarity index 84% rename from mixin/thanos/alerts/receiver.libsonnet rename to mixin/thanos/alerts/receive.libsonnet index 9441bcaf41..a5dd02f624 100644 --- a/mixin/thanos/alerts/receiver.libsonnet +++ b/mixin/thanos/alerts/receive.libsonnet @@ -1,16 +1,16 @@ { local thanos = self, - receiver+:: { + receive+:: { jobPrefix: error 'must provide job prefix for Thanos Receive alerts', selector: error 'must provide selector for Thanos Receive alerts', }, prometheusAlerts+:: { groups+: [ { - name: 'thanos-receiver.rules', + name: 'thanos-receive.rules', rules: [ { - alert: 'ThanosReceiverHttpRequestErrorRateHigh', + alert: 'ThanosReceiveHttpRequestErrorRateHigh', annotations: { message: 'Thanos Receive {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', }, @@ -20,14 +20,14 @@ / sum(rate(http_requests_total{%(selector)s, handler="receive"}[5m])) ) * 100 > 5 - ||| % thanos.receiver, + ||| % thanos.receive, 'for': '5m', labels: { severity: 'critical', }, }, { - alert: 'ThanosReceiverHttpRequestLatencyHigh', + alert: 'ThanosReceiveHttpRequestLatencyHigh', annotations: { message: 'Thanos Receive {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for requests.', }, @@ -37,14 +37,14 @@ and sum by (job) (rate(http_request_duration_seconds_count{%(selector)s, handler="receive"}[5m])) > 0 ) - ||| % thanos.receiver, + ||| % thanos.receive, 'for': '10m', labels: { severity: 'critical', }, }, { - alert: 'ThanosReceiverHighForwardRequestFailures', + alert: 'ThanosReceiveHighForwardRequestFailures', annotations: { message: 'Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize }}% of requests.', }, @@ -55,14 +55,14 @@ sum by (job) (rate(thanos_receive_forward_requests_total{%(selector)s}[5m])) * 100 > 5 ) - ||| % thanos.receiver, + ||| % thanos.receive, 'for': '5m', labels: { severity: 'critical', }, }, { - alert: 'ThanosReceiverHighHashringFileRefreshFailures', + alert: 'ThanosReceiveHighHashringFileRefreshFailures', annotations: { message: 'Thanos Receive {{$labels.job}} is failing to refresh hashring file, {{ $value | humanize }} of attempts failed.', }, @@ -73,18 +73,18 @@ sum by (job) (rate(thanos_receive_hashrings_file_refreshes_total{%(selector)s}[5m])) > 0 ) - ||| % thanos.receiver, + ||| % thanos.receive, 'for': '15m', labels: { severity: 'warning', }, }, { - alert: 'ThanosReceiverConfigReloadFailure', + alert: 'ThanosReceiveConfigReloadFailure', annotations: { message: 'Thanos Receive {{$labels.job}} has not been able to reload hashring configurations.', }, - expr: 'avg(thanos_receive_config_last_reload_successful{%(selector)s}) by (job) != 1' % thanos.receiver, + expr: 'avg(thanos_receive_config_last_reload_successful{%(selector)s}) by (job) != 1' % thanos.receive, 'for': '5m', labels: { severity: 'warning', diff --git a/mixin/thanos/alerts/ruler.libsonnet b/mixin/thanos/alerts/rule.libsonnet similarity index 63% rename from mixin/thanos/alerts/ruler.libsonnet rename to mixin/thanos/alerts/rule.libsonnet index 83f4d59807..d7eecdcbc3 100644 --- a/mixin/thanos/alerts/ruler.libsonnet +++ b/mixin/thanos/alerts/rule.libsonnet @@ -1,44 +1,44 @@ { local thanos = self, - ruler+:: { - jobPrefix: error 'must provide job prefix for Thanos Ruler alerts', - selector: error 'must provide selector for Thanos Ruler alerts', + rule+:: { + jobPrefix: error 'must provide job prefix for Thanos Rule alerts', + selector: error 'must provide selector for Thanos Rule alerts', }, prometheusAlerts+:: { groups+: [ { - name: 'thanos-ruler.rules', + name: 'thanos-rule.rules', rules: [ { - alert: 'ThanosRulerQueueIsDroppingAlerts', + alert: 'ThanosRuleQueueIsDroppingAlerts', annotations: { - message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to queue alerts.', + message: 'Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to queue alerts.', }, expr: ||| sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{%(selector)s}[5m])) > 0 - ||| % thanos.ruler, + ||| % thanos.rule, 'for': '5m', labels: { severity: 'critical', }, }, { - alert: 'ThanosRulerSenderIsFailingAlerts', + alert: 'ThanosRuleSenderIsFailingAlerts', annotations: { - message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to send alerts to alertmanager.', + message: 'Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to send alerts to alertmanager.', }, expr: ||| sum by (job) (rate(thanos_alert_sender_alerts_dropped_total{%(selector)s}[5m])) > 0 - ||| % thanos.ruler, + ||| % thanos.rule, 'for': '5m', labels: { severity: 'critical', }, }, { - alert: 'ThanosRulerHighRuleEvaluationFailures', + alert: 'ThanosRuleHighRuleEvaluationFailures', annotations: { - message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} is failing to evaluate rules.', + message: 'Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to evaluate rules.', }, expr: ||| ( @@ -47,7 +47,7 @@ sum by (job) (rate(prometheus_rule_evaluations_total{%(selector)s}[5m])) * 100 > 5 ) - ||| % thanos.ruler, + ||| % thanos.rule, 'for': '5m', labels: { @@ -55,13 +55,13 @@ }, }, { - alert: 'ThanosRulerHighRuleEvaluationWarnings', + alert: 'ThanosRuleHighRuleEvaluationWarnings', annotations: { - message: 'Thanos Ruler {{$labels.job}} {{$labels.pod}} has high number of evaluation warnings.', + message: 'Thanos Rule {{$labels.job}} {{$labels.pod}} has high number of evaluation warnings.', }, expr: ||| sum by (job) (rate(thanos_rule_evaluation_with_warnings_total{%(selector)s}[5m])) > 0 - ||| % thanos.ruler, + ||| % thanos.rule, 'for': '15m', labels: { @@ -69,9 +69,9 @@ }, }, { - alert: 'ThanosRulerRuleEvaluationLatencyHigh', + alert: 'ThanosRuleRuleEvaluationLatencyHigh', annotations: { - message: 'Thanos Ruler {{$labels.job}}/{{$labels.pod}} has higher evaluation latency than interval for {{$labels.rule_group}}.', + message: 'Thanos Rule {{$labels.job}}/{{$labels.pod}} has higher evaluation latency than interval for {{$labels.rule_group}}.', }, expr: ||| ( @@ -79,16 +79,16 @@ > sum by (job, pod, rule_group) (prometheus_rule_group_interval_seconds{%(selector)s}) ) - ||| % thanos.ruler, + ||| % thanos.rule, 'for': '5m', labels: { severity: 'warning', }, }, { - alert: 'ThanosRulerGrpcErrorRate', + alert: 'ThanosRuleGrpcErrorRate', annotations: { - message: 'Thanos Ruler {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', + message: 'Thanos Rule {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.', }, expr: ||| ( @@ -97,27 +97,27 @@ sum by (job) (rate(grpc_server_started_total{%(selector)s}[5m])) * 100 > 5 ) - ||| % thanos.ruler, + ||| % thanos.rule, 'for': '5m', labels: { severity: 'warning', }, }, { - alert: 'ThanosRulerConfigReloadFailure', + alert: 'ThanosRuleConfigReloadFailure', annotations: { - message: 'Thanos Ruler {{$labels.job}} has not been able to reload its configuration.', + message: 'Thanos Rule {{$labels.job}} has not been able to reload its configuration.', }, - expr: 'avg(thanos_rule_config_last_reload_successful{%(selector)s}) by (job) != 1' % thanos.ruler, + expr: 'avg(thanos_rule_config_last_reload_successful{%(selector)s}) by (job) != 1' % thanos.rule, 'for': '5m', labels: { severity: 'warning', }, }, { - alert: 'ThanosRulerQueryHighDNSFailures', + alert: 'ThanosRuleQueryHighDNSFailures', annotations: { - message: 'Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for query endpoints.', + message: 'Thanos Rule {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for query endpoints.', }, expr: ||| ( @@ -126,16 +126,16 @@ sum by (job) (rate(thanos_ruler_query_apis_dns_lookups_total{%(selector)s}[5m])) * 100 > 1 ) - ||| % thanos.ruler, + ||| % thanos.rule, 'for': '15m', labels: { severity: 'warning', }, }, { - alert: 'ThanosRulerAlertmanagerHighDNSFailures', + alert: 'ThanosRuleAlertmanagerHighDNSFailures', annotations: { - message: 'Thanos Ruler {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for Alertmanager endpoints.', + message: 'Thanos Rule {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for Alertmanager endpoints.', }, expr: ||| ( @@ -144,7 +144,7 @@ sum by (job) (rate(thanos_ruler_alertmanagers_dns_lookups_total{%(selector)s}[5m])) * 100 > 1 ) - ||| % thanos.ruler, + ||| % thanos.rule, 'for': '15m', labels: { severity: 'warning', diff --git a/mixin/thanos/dashboards/compactor.libsonnet b/mixin/thanos/dashboards/compact.libsonnet similarity index 91% rename from mixin/thanos/dashboards/compactor.libsonnet rename to mixin/thanos/dashboards/compact.libsonnet index 35e9c18c61..dbd30f926d 100644 --- a/mixin/thanos/dashboards/compactor.libsonnet +++ b/mixin/thanos/dashboards/compact.libsonnet @@ -2,14 +2,14 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, - compactor+:: { - jobPrefix: error 'must provide job prefix for Thanos Compactor dashboard', - selector: error 'must provide selector for Thanos Compactor dashboard', - title: error 'must provide title for Thanos Compactor dashboard', + compact+:: { + jobPrefix: error 'must provide job prefix for Thanos Compact dashboard', + selector: error 'must provide selector for Thanos Compact dashboard', + title: error 'must provide title for Thanos Compact dashboard', }, grafanaDashboards+:: { - 'compactor.json': - g.dashboard(thanos.compactor.title) + 'compact.json': + g.dashboard(thanos.compact.title) .addRow( g.row('Group Compaction') .addPanel( @@ -131,22 +131,22 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; g.resourceUtilizationRow() ) + g.template('namespace', thanos.dashboard.namespaceQuery) + - g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.compactor, true, '%(jobPrefix)s.*' % thanos.compactor) + - g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.compactor, true, '.*'), + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.compact, true, '%(jobPrefix)s.*' % thanos.compact) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.compact, true, '.*'), __overviewRows__+:: [ - g.row('Compactor') + g.row('Compact') .addPanel( g.panel( 'Compaction Rate', 'Shows rate of execution for compactions against blocks that are stored in the bucket by compaction group.' ) + g.queryPanel( - 'sum(rate(thanos_compact_group_compactions_total{namespace="$namespace",%(selector)s}[$interval])) by (job)' % thanos.compactor, + 'sum(rate(thanos_compact_group_compactions_total{namespace="$namespace",%(selector)s}[$interval])) by (job)' % thanos.compact, 'compaction {{job}}' ) + g.stack + - g.addDashboardLink(thanos.compactor.title) + g.addDashboardLink(thanos.compact.title) ) .addPanel( g.panel( @@ -154,10 +154,10 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; 'Shows ratio of errors compared to the total number of executed compactions against blocks that are stored in the bucket.' ) + g.qpsErrTotalPanel( - 'thanos_compact_group_compactions_failures_total{namespace="$namespace",%(selector)s}' % thanos.compactor, - 'thanos_compact_group_compactions_total{namespace="$namespace",%(selector)s}' % thanos.compactor, + 'thanos_compact_group_compactions_failures_total{namespace="$namespace",%(selector)s}' % thanos.compact, + 'thanos_compact_group_compactions_total{namespace="$namespace",%(selector)s}' % thanos.compact, ) + - g.addDashboardLink(thanos.compactor.title) + g.addDashboardLink(thanos.compact.title) ) + g.collapse, ], diff --git a/mixin/thanos/dashboards/dashboards.libsonnet b/mixin/thanos/dashboards/dashboards.libsonnet index f12f527103..67fccfcae7 100644 --- a/mixin/thanos/dashboards/dashboards.libsonnet +++ b/mixin/thanos/dashboards/dashboards.libsonnet @@ -1,8 +1,8 @@ -(import 'querier.libsonnet') + +(import 'query.libsonnet') + (import 'store.libsonnet') + (import 'sidecar.libsonnet') + -(import 'receiver.libsonnet') + -(import 'ruler.libsonnet') + -(import 'compactor.libsonnet') + +(import 'receive.libsonnet') + +(import 'rule.libsonnet') + +(import 'compact.libsonnet') + (import 'overview.libsonnet') + (import 'defaults.libsonnet') diff --git a/mixin/thanos/dashboards/querier.libsonnet b/mixin/thanos/dashboards/query.libsonnet similarity index 90% rename from mixin/thanos/dashboards/querier.libsonnet rename to mixin/thanos/dashboards/query.libsonnet index 18ef95c064..d390f8c4d6 100644 --- a/mixin/thanos/dashboards/querier.libsonnet +++ b/mixin/thanos/dashboards/query.libsonnet @@ -2,14 +2,14 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, - querier+:: { + query+:: { jobPrefix: error 'must provide job prefix for Thanos Query dashboard', selector: error 'must provide selector for Thanos Query dashboard', title: error 'must provide title for Thanos Query dashboard', }, grafanaDashboards+:: { - 'querier.json': - g.dashboard(thanos.querier.title) + 'query.json': + g.dashboard(thanos.query.title) .addRow( g.row('Instant Query API') .addPanel( @@ -139,54 +139,54 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; g.resourceUtilizationRow() ) + g.template('namespace', thanos.dashboard.namespaceQuery) + - g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.querier, true, '%(jobPrefix)s.*' % thanos.querier) + - g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.querier, true, '.*'), + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.query, true, '%(jobPrefix)s.*' % thanos.query) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.query, true, '.*'), __overviewRows__+:: [ g.row('Instant Query') .addPanel( g.panel('Requests Rate', 'Shows rate of requests against /query for the given time.') + - g.httpQpsPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query"' % thanos.querier) + - g.addDashboardLink(thanos.querier.title) + g.httpQpsPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query"' % thanos.query) + + g.addDashboardLink(thanos.query.title) ) .addPanel( g.panel('Requests Errors', 'Shows ratio of errors compared to the the total number of handled requests against /query.') + - g.httpErrPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query"' % thanos.querier) + - g.addDashboardLink(thanos.querier.title) + g.httpErrPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query"' % thanos.query) + + g.addDashboardLink(thanos.query.title) ) .addPanel( g.sloLatency( 'Latency 99th Percentile', 'Shows how long has it taken to handle requests.', - 'http_request_duration_seconds_bucket{namespace="$namespace",%(selector)s,handler="query"}' % thanos.querier, + 'http_request_duration_seconds_bucket{namespace="$namespace",%(selector)s,handler="query"}' % thanos.query, 0.99, 0.5, 1 ) + - g.addDashboardLink(thanos.querier.title) + g.addDashboardLink(thanos.query.title) ), g.row('Range Query') .addPanel( g.panel('Requests Rate', 'Shows rate of requests against /query_range for the given time range.') + - g.httpQpsPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query_range"' % thanos.querier) + - g.addDashboardLink(thanos.querier.title) + g.httpQpsPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query_range"' % thanos.query) + + g.addDashboardLink(thanos.query.title) ) .addPanel( g.panel('Requests Errors', 'Shows ratio of errors compared to the the total number of handled requests against /query_range.') + - g.httpErrPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query_range"' % thanos.querier) + - g.addDashboardLink(thanos.querier.title) + g.httpErrPanel('http_requests_total', 'namespace="$namespace",%(selector)s,handler="query_range"' % thanos.query) + + g.addDashboardLink(thanos.query.title) ) .addPanel( g.sloLatency( 'Latency 99th Percentile', 'Shows how long has it taken to handle requests.', - 'http_request_duration_seconds_bucket{namespace="$namespace",%(selector)s,handler="query_range"}' % thanos.querier, + 'http_request_duration_seconds_bucket{namespace="$namespace",%(selector)s,handler="query_range"}' % thanos.query, 0.99, 0.5, 1 ) + - g.addDashboardLink(thanos.querier.title) + g.addDashboardLink(thanos.query.title) ), ], }, diff --git a/mixin/thanos/dashboards/receiver.libsonnet b/mixin/thanos/dashboards/receive.libsonnet similarity index 93% rename from mixin/thanos/dashboards/receiver.libsonnet rename to mixin/thanos/dashboards/receive.libsonnet index dd35a6ee9f..1d8346a20b 100644 --- a/mixin/thanos/dashboards/receiver.libsonnet +++ b/mixin/thanos/dashboards/receive.libsonnet @@ -2,14 +2,14 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, - receiver+:: { + receive+:: { jobPrefix: error 'must provide job prefix for Thanos Receive dashboard', selector: error 'must provide selector for Thanos Receive dashboard', title: error 'must provide title for Thanos Receive dashboard', }, grafanaDashboards+:: { - 'receiver.json': - g.dashboard(thanos.receiver.title) + 'receive.json': + g.dashboard(thanos.receive.title) .addRow( g.row('Incoming Request') .addPanel( @@ -140,31 +140,31 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; g.resourceUtilizationRow() ) + g.template('namespace', thanos.dashboard.namespaceQuery) + - g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.receiver, true, '%(jobPrefix)s.*' % thanos.receiver) + - g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.receiver, true, '.*'), + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.receive, true, '%(jobPrefix)s.*' % thanos.receive) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.receive, true, '.*'), __overviewRows__+:: [ g.row('Receive') .addPanel( g.panel('Incoming Requests Rate', 'Shows rate of incoming requests.') + - g.httpQpsPanel('http_requests_total', 'handler="receive",namespace="$namespace",%(selector)s' % thanos.receiver) + - g.addDashboardLink(thanos.receiver.title) + g.httpQpsPanel('http_requests_total', 'handler="receive",namespace="$namespace",%(selector)s' % thanos.receive) + + g.addDashboardLink(thanos.receive.title) ) .addPanel( g.panel('Incoming Requests Errors', 'Shows ratio of errors compared to the total number of handled incoming requests.') + - g.httpErrPanel('http_requests_total', 'handler="receive",namespace="$namespace",%(selector)s' % thanos.receiver) + - g.addDashboardLink(thanos.receiver.title) + g.httpErrPanel('http_requests_total', 'handler="receive",namespace="$namespace",%(selector)s' % thanos.receive) + + g.addDashboardLink(thanos.receive.title) ) .addPanel( g.sloLatency( 'Incoming Requests Latency 99th Percentile', 'Shows how long has it taken to handle incoming requests.', - 'http_request_duration_seconds_bucket{handler="receive",namespace="$namespace",%(selector)s}' % thanos.receiver, + 'http_request_duration_seconds_bucket{handler="receive",namespace="$namespace",%(selector)s}' % thanos.receive, 0.99, 0.5, 1 ) + - g.addDashboardLink(thanos.receiver.title) + g.addDashboardLink(thanos.receive.title) ), ], }, diff --git a/mixin/thanos/dashboards/ruler.libsonnet b/mixin/thanos/dashboards/rule.libsonnet similarity index 90% rename from mixin/thanos/dashboards/ruler.libsonnet rename to mixin/thanos/dashboards/rule.libsonnet index 067e4d1af2..d172b9f2b3 100644 --- a/mixin/thanos/dashboards/ruler.libsonnet +++ b/mixin/thanos/dashboards/rule.libsonnet @@ -2,14 +2,14 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; { local thanos = self, - ruler+:: { - jobPrefix: error 'must provide job prefix for Thanos Ruler dashboard', - selector: error 'must provide selector for Thanos Ruler dashboard', - title: error 'must provide title for Thanos Ruler dashboard', + rule+:: { + jobPrefix: error 'must provide job prefix for Thanos Rule dashboard', + selector: error 'must provide selector for Thanos Rule dashboard', + title: error 'must provide title for Thanos Rule dashboard', }, grafanaDashboards+:: { - 'ruler.json': - g.dashboard(thanos.ruler.title) + 'rule.json': + g.dashboard(thanos.rule.title) .addRow( g.row('Alert Sent') .addPanel( @@ -122,38 +122,38 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet'; g.resourceUtilizationRow() ) + g.template('namespace', thanos.dashboard.namespaceQuery) + - g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.ruler, true, '%(jobPrefix)s.*' % thanos.ruler) + - g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.ruler, true, '.*'), + g.template('job', 'up', 'namespace="$namespace",%(selector)s' % thanos.rule, true, '%(jobPrefix)s.*' % thanos.rule) + + g.template('pod', 'kube_pod_info', 'namespace="$namespace",created_by_name=~"%(jobPrefix)s.*"' % thanos.rule, true, '.*'), __overviewRows__+:: [ g.row('Rule') .addPanel( g.panel('Alert Sent Rate', 'Shows rate of alerts that successfully sent to alert manager.') + g.queryPanel( - 'sum(rate(thanos_alert_sender_alerts_sent_total{namespace="$namespace",%(selector)s}[$interval])) by (job, alertmanager)' % thanos.ruler, + 'sum(rate(thanos_alert_sender_alerts_sent_total{namespace="$namespace",%(selector)s}[$interval])) by (job, alertmanager)' % thanos.rule, '{{job}} {{alertmanager}}' ) + - g.addDashboardLink(thanos.ruler.title) + + g.addDashboardLink(thanos.rule.title) + g.stack ) .addPanel( g.panel('Alert Sent Errors', 'Shows ratio of errors compared to the total number of sent alerts.') + g.qpsErrTotalPanel( - 'thanos_alert_sender_errors_total{namespace="$namespace",%(selector)s}' % thanos.ruler, - 'thanos_alert_sender_alerts_sent_total{namespace="$namespace",%(selector)s}' % thanos.ruler, + 'thanos_alert_sender_errors_total{namespace="$namespace",%(selector)s}' % thanos.rule, + 'thanos_alert_sender_alerts_sent_total{namespace="$namespace",%(selector)s}' % thanos.rule, ) + - g.addDashboardLink(thanos.ruler.title) + g.addDashboardLink(thanos.rule.title) ) .addPanel( g.sloLatency( 'Alert Sent Duration', 'Shows how long has it taken to send alerts to alert manager.', - 'thanos_alert_sender_latency_seconds_bucket{namespace="$namespace",%(selector)s}' % thanos.ruler, + 'thanos_alert_sender_latency_seconds_bucket{namespace="$namespace",%(selector)s}' % thanos.rule, 0.99, 0.5, 1 ) + - g.addDashboardLink(thanos.ruler.title) + g.addDashboardLink(thanos.rule.title) ) + g.collapse, ], diff --git a/mixin/thanos/defaults.libsonnet b/mixin/thanos/defaults.libsonnet index a6cf5e9cdc..b2a0c9d76a 100644 --- a/mixin/thanos/defaults.libsonnet +++ b/mixin/thanos/defaults.libsonnet @@ -1,28 +1,28 @@ { - querier+:: { - jobPrefix: 'thanos-querier', + query+:: { + jobPrefix: 'thanos-query', selector: 'job=~"%s.*"' % self.jobPrefix, - title: '%(prefix)sQuerier' % $.dashboard.prefix, + title: '%(prefix)sQuery' % $.dashboard.prefix, }, store+:: { jobPrefix: 'thanos-store', selector: 'job=~"%s.*"' % self.jobPrefix, title: '%(prefix)sStore' % $.dashboard.prefix, }, - receiver+:: { - jobPrefix: 'thanos-receiver', + receive+:: { + jobPrefix: 'thanos-receive', selector: 'job=~"%s.*"' % self.jobPrefix, - title: '%(prefix)sReceiver' % $.dashboard.prefix, + title: '%(prefix)sReceive' % $.dashboard.prefix, }, - ruler+:: { - jobPrefix: 'thanos-ruler', + rule+:: { + jobPrefix: 'thanos-rule', selector: 'job=~"%s.*"' % self.jobPrefix, - title: '%(prefix)sRuler' % $.dashboard.prefix, + title: '%(prefix)sRule' % $.dashboard.prefix, }, - compactor+:: { - jobPrefix: 'thanos-compactor', + compact+:: { + jobPrefix: 'thanos-compact', selector: 'job=~"%s.*"' % self.jobPrefix, - title: '%(prefix)sCompactor' % $.dashboard.prefix, + title: '%(prefix)sCompact' % $.dashboard.prefix, }, sidecar+:: { jobPrefix: 'thanos-sidecar', diff --git a/mixin/thanos/rules/querier.libsonnet b/mixin/thanos/rules/query.libsonnet similarity index 90% rename from mixin/thanos/rules/querier.libsonnet rename to mixin/thanos/rules/query.libsonnet index 22706cf0e5..9e7e25c1cf 100644 --- a/mixin/thanos/rules/querier.libsonnet +++ b/mixin/thanos/rules/query.libsonnet @@ -1,12 +1,12 @@ { local thanos = self, - querier+:: { + query+:: { selector: error 'must provide selector for Thanos Query recording rules', }, prometheusRules+:: { groups+: [ { - name: 'thanos-querier.rules', + name: 'thanos-query.rules', rules: [ { record: ':grpc_client_failures_per_unary:sum_rate', @@ -16,7 +16,7 @@ / sum(rate(grpc_client_started_total{%(selector)s, grpc_type="unary"}[5m])) ) - ||| % thanos.querier, + ||| % thanos.query, labels: { }, }, @@ -28,7 +28,7 @@ / sum(rate(grpc_client_started_total{%(selector)s, grpc_type="server_stream"}[5m])) ) - ||| % thanos.querier, + ||| % thanos.query, labels: { }, }, @@ -40,7 +40,7 @@ / sum(rate(thanos_querier_store_apis_dns_lookups_total{%(selector)s}[5m])) ) - ||| % thanos.querier, + ||| % thanos.query, labels: { }, }, @@ -50,7 +50,7 @@ histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{%(selector)s, handler="query"}[5m])) by (le) ) - ||| % thanos.querier, + ||| % thanos.query, labels: { quantile: '0.99', }, @@ -61,7 +61,7 @@ histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{%(selector)s, handler="query_range"}[5m])) by (le) ) - ||| % thanos.querier, + ||| % thanos.query, labels: { quantile: '0.99', }, diff --git a/mixin/thanos/rules/receiver.libsonnet b/mixin/thanos/rules/receive.libsonnet similarity index 90% rename from mixin/thanos/rules/receiver.libsonnet rename to mixin/thanos/rules/receive.libsonnet index fc01667e58..d6e11558cd 100644 --- a/mixin/thanos/rules/receiver.libsonnet +++ b/mixin/thanos/rules/receive.libsonnet @@ -1,12 +1,12 @@ { local thanos = self, - receiver+:: { + receive+:: { selector: error 'must provide selector for Thanos Receive recording rules', }, prometheusRules+:: { groups+: [ { - name: 'thanos-receiver.rules', + name: 'thanos-receive.rules', rules: [ { record: ':grpc_server_failures_per_unary:sum_rate', @@ -16,7 +16,7 @@ / rate(grpc_server_started_total{%(selector)s, grpc_type="unary"}[5m]) ) - ||| % thanos.receiver, + ||| % thanos.receive, labels: { }, }, @@ -28,7 +28,7 @@ / rate(grpc_server_started_total{%(selector)s, grpc_type="server_stream"}[5m]) ) - ||| % thanos.receiver, + ||| % thanos.receive, labels: { }, }, @@ -40,7 +40,7 @@ / rate(http_requests_total{handler="receive", %(selector)s}[5m]) ) - ||| % thanos.receiver, + ||| % thanos.receive, labels: { }, }, @@ -50,7 +50,7 @@ histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{handler="receive", %(selector)s}[5m])) by (le) ) - ||| % thanos.receiver, + ||| % thanos.receive, labels: { quantile: '0.99', }, @@ -63,7 +63,7 @@ / sum(rate(thanos_receive_forward_requests_total{%(selector)s}[5m])) ) - ||| % thanos.receiver, + ||| % thanos.receive, labels: { }, }, @@ -75,7 +75,7 @@ / sum(rate(thanos_receive_hashrings_file_refreshes_total{%(selector)s}[5m])) ) - ||| % thanos.receiver, + ||| % thanos.receive, labels: { }, }, diff --git a/mixin/thanos/rules/rules.libsonnet b/mixin/thanos/rules/rules.libsonnet index 656b267b4f..c74492d4d4 100644 --- a/mixin/thanos/rules/rules.libsonnet +++ b/mixin/thanos/rules/rules.libsonnet @@ -1,3 +1,3 @@ -(import 'querier.libsonnet') + -(import 'receiver.libsonnet') + +(import 'query.libsonnet') + +(import 'receive.libsonnet') + (import 'store.libsonnet') From 88004d11c11ccad1051af87e83beeb8d4108c7b9 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 17 Feb 2020 12:00:53 +0100 Subject: [PATCH 245/257] tracing: track query sent to prometheus via remote read api (#2145) * tracing: track query sent to prometheus via remote read api Signed-off-by: Igor Wiedler * add CHANGELOG entry Signed-off-by: Igor Wiedler --- CHANGELOG.md | 1 + pkg/store/prometheus.go | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a7737f4d8..bc68770465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#2030](https://github.com/thanos-io/thanos/pull/2030) Query: Add `thanos_proxy_store_empty_stream_responses_total` metric for number of empty responses from stores. - [#2049](https://github.com/thanos-io/thanos/pull/2049) Tracing: Support sampling on Elastic APM with new sample_rate setting. - [#2008](https://github.com/thanos-io/thanos/pull/2008) Querier, Receiver, Sidecar, Store: Add gRPC [health check](https://github.com/grpc/grpc/blob/master/doc/health-checking.md) endpoints. +- [#2145](https://github.com/thanos-io/thanos/pull/2145) Tracing: track query sent to prometheus via remote read api. ### Changed diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 7d8c762421..ed32b389b5 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -432,6 +432,11 @@ func (p *PrometheusStore) startPromSeries(ctx context.Context, q *prompb.Query) return nil, errors.Wrap(err, "marshal read request") } + qjson, err := json.Marshal(q) + if err != nil { + return nil, errors.Wrap(err, "json encode query for tracing") + } + u := *p.base u.Path = path.Join(u.Path, "api/v1/read") @@ -442,13 +447,13 @@ func (p *PrometheusStore) startPromSeries(ctx context.Context, q *prompb.Query) preq.Header.Add("Content-Encoding", "snappy") preq.Header.Set("Content-Type", "application/x-stream-protobuf") preq.Header.Set("User-Agent", userAgent) - spanReqDo, ctx := tracing.StartSpan(ctx, "query_prometheus_request") + spanReqDo, ctx := tracing.StartSpan(ctx, "query_prometheus_request", opentracing.Tag{Key: "prometheus.query", Value: string(qjson)}) preq = preq.WithContext(ctx) presp, err := p.client.Do(preq) + spanReqDo.Finish() if err != nil { return nil, errors.Wrap(err, "send request") } - spanReqDo.Finish() if presp.StatusCode/100 != 2 { // Best effort read. b, err := ioutil.ReadAll(presp.Body) From 01cee873a6adac635c59a0eb11b084b07d0e2972 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Mon, 17 Feb 2020 16:45:52 +0100 Subject: [PATCH 246/257] *: Handle SIGHUP (#2139) * Handle SIGHUP Signed-off-by: Kemal Akkoyun * Only log when signal received Signed-off-by: Kemal Akkoyun * Use a buffered channel Signed-off-by: Kemal Akkoyun --- cmd/thanos/bucket.go | 8 ++++---- cmd/thanos/check.go | 2 +- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/main.go | 35 +++++++++++++++++++++++++++++++++-- cmd/thanos/query.go | 2 +- cmd/thanos/receive.go | 2 +- cmd/thanos/rule.go | 29 ++++------------------------- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- 10 files changed, 48 insertions(+), 38 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 1aed48e781..6967229b05 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -80,7 +80,7 @@ func registerBucketVerify(m map[string]setupFunc, root *kingpin.CmdClause, name Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() idWhitelist := cmd.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+ "If none is specified, all blocks will be verified. Repeated field").Strings() - m[name+" verify"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { + m[name+" verify"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error { confContentYaml, err := objStoreConfig.Content() if err != nil { return err @@ -167,7 +167,7 @@ func registerBucketLs(m map[string]setupFunc, root *kingpin.CmdClause, name stri cmd := root.Command("ls", "List all blocks in the bucket") output := cmd.Flag("output", "Optional format in which to print each block's information. Options are 'json', 'wide' or a custom template."). Short('o').Default("").String() - m[name+" ls"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { + m[name+" ls"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error { confContentYaml, err := objStoreConfig.Content() if err != nil { return err @@ -260,7 +260,7 @@ func registerBucketInspect(m map[string]setupFunc, root *kingpin.CmdClause, name Default("FROM", "UNTIL").Enums(inspectColumns...) timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() - m[name+" inspect"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { + m[name+" inspect"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error { // Parse selector. selectorLabels, err := parseFlagLabels(*selector) @@ -316,7 +316,7 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() label := cmd.Flag("label", "Prometheus label to use as timeline title").String() - m[name+" web"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { + m[name+" web"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error { ctx, cancel := context.WithCancel(context.Background()) comp := component.Bucket diff --git a/cmd/thanos/check.go b/cmd/thanos/check.go index d42c12052c..ae0d43fa27 100644 --- a/cmd/thanos/check.go +++ b/cmd/thanos/check.go @@ -29,7 +29,7 @@ func registerCheckRules(m map[string]setupFunc, root *kingpin.CmdClause, name st "The rule files to check.", ).Required().ExistingFiles() - m[name+" rules"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { + m[name+" rules"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error { // Dummy actor to immediately kill the group after the run function returns. g.Add(func() error { return nil }, func(error) {}) return checkRulesFiles(logger, ruleFiles) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 4bdd9cc351..4f4b9a578f 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -124,7 +124,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) { selectorRelabelConf := regSelectorRelabelFlags(cmd) - m[component.Compact.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { + m[component.Compact.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { return runCompact(g, logger, reg, *httpAddr, time.Duration(*httpGracePeriod), diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index b7b65369fa..d38ea26273 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -44,7 +44,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application) { objStoreConfig := regCommonObjStoreFlags(cmd, "", true) - m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { + m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { return runDownsample(g, logger, reg, *httpAddr, time.Duration(*httpGracePeriod), *dataDir, objStoreConfig, comp) } } diff --git a/cmd/thanos/main.go b/cmd/thanos/main.go index 9c38acca4b..f77424d8ce 100644 --- a/cmd/thanos/main.go +++ b/cmd/thanos/main.go @@ -32,7 +32,7 @@ const ( logFormatJson = "json" ) -type setupFunc func(*run.Group, log.Logger, *prometheus.Registry, opentracing.Tracer, bool) error +type setupFunc func(*run.Group, log.Logger, *prometheus.Registry, opentracing.Tracer, <-chan struct{}, bool) error func main() { if os.Getenv("DEBUG") != "" { @@ -178,7 +178,10 @@ func main() { }) } - if err := cmds[cmd](&g, logger, metrics, tracer, *logLevel == "debug"); err != nil { + // Create a signal channel to dispatch reload events to sub-commands. + reloadCh := make(chan struct{}, 1) + + if err := cmds[cmd](&g, logger, metrics, tracer, reloadCh, *logLevel == "debug"); err != nil { level.Error(logger).Log("err", errors.Wrapf(err, "%s command failed", cmd)) os.Exit(1) } @@ -193,6 +196,16 @@ func main() { }) } + // Listen for reload signals. + { + cancel := make(chan struct{}) + g.Add(func() error { + return reload(logger, cancel, reloadCh) + }, func(error) { + close(cancel) + }) + } + if err := g.Run(); err != nil { level.Error(logger).Log("msg", "running command failed", "err", err) os.Exit(1) @@ -211,3 +224,21 @@ func interrupt(logger log.Logger, cancel <-chan struct{}) error { return errors.New("canceled") } } + +func reload(logger log.Logger, cancel <-chan struct{}, r chan<- struct{}) error { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGHUP) + for { + select { + case s := <-c: + level.Info(logger).Log("msg", "caught signal. Reloading.", "signal", s) + select { + case r <- struct{}{}: + level.Info(logger).Log("msg", "relaod dispatched.") + default: + } + case <-cancel: + return errors.New("canceled") + } + } +} diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index f64584b27d..9433f5601d 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -101,7 +101,7 @@ func registerQuery(m map[string]setupFunc, app *kingpin.Application) { storeResponseTimeout := modelDuration(cmd.Flag("store.response-timeout", "If a Store doesn't send any data in this specified duration then a Store will be ignored and partial data will be returned if it's enabled. 0 disables timeout.").Default("0ms")) - m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { + m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { selectorLset, err := parseFlagLabels(*selectorLabels) if err != nil { return errors.Wrap(err, "parse federation labels") diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 6f5e238618..21fa243805 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -81,7 +81,7 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) { walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool() - m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { + m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { lset, err := parseFlagLabels(*labelStrs) if err != nil { return errors.Wrap(err, "parse labels") diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 7357f7ae48..b275ba8b36 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -8,13 +8,10 @@ import ( "math/rand" "net/http" "net/url" - "os" - "os/signal" "path" "path/filepath" "strconv" "strings" - "syscall" "time" "github.com/go-kit/kit/log" @@ -116,7 +113,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { dnsSDResolver := cmd.Flag("query.sd-dns-resolver", "Resolver to use. Possible options: [golang, miekgdns]"). Default("golang").Hidden().String() - m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { + m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, reload <-chan struct{}, _ bool) error { lset, err := parseFlagLabels(*labelStrs) if err != nil { return errors.Wrap(err, "parse labels") @@ -168,6 +165,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) { logger, reg, tracer, + reload, lset, *alertmgrs, *alertmgrsTimeout, @@ -209,6 +207,7 @@ func runRule( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, + reloadSignal <-chan struct{}, lset labels.Labels, alertmgrURLs []string, alertmgrsTimeout time.Duration, @@ -484,6 +483,7 @@ func runRule( case <-cancel: return errors.New("canceled") case <-reload: + case <-reloadSignal: } level.Debug(logger).Log("msg", "configured rule files", "files", strings.Join(ruleFiles, ",")) @@ -520,27 +520,6 @@ func runRule( close(cancel) }) } - { - cancel := make(chan struct{}) - - g.Add(func() error { - c := make(chan os.Signal, 1) - for { - signal.Notify(c, syscall.SIGHUP) - select { - case <-c: - select { - case reload <- struct{}{}: - default: - } - case <-cancel: - return errors.New("canceled") - } - } - }, func(error) { - close(cancel) - }) - } grpcProbe := prober.NewGRPC() httpProbe := prober.NewHTTP() diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 14d559c49f..3780ad1812 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -75,7 +75,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { minTime := thanosmodel.TimeOrDuration(cmd.Flag("min-time", "Start of time range limit to serve. Thanos sidecar will serve only metrics, which happened later than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y."). Default("0000-01-01T00:00:00Z")) - m[component.Sidecar.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { + m[component.Sidecar.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { rl := reloader.New( log.With(logger, "component", "reloader"), reloader.ReloadURLFromBase(*promURL), diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index bb2a547d68..e85f80e67d 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -81,7 +81,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { enableIndexHeader := cmd.Flag("experimental.enable-index-header", "If true, Store Gateway will recreate index-header instead of index-cache.json for each block. This will replace index-cache.json permanently once it will be out of experimental stage."). Hidden().Default("false").Bool() - m[component.Store.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, debugLogging bool) error { + m[component.Store.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, debugLogging bool) error { if minTime.PrometheusTimestamp() > maxTime.PrometheusTimestamp() { return errors.Errorf("invalid argument: --min-time '%s' can't be greater than --max-time '%s'", minTime, maxTime) From 2e3ece187f8fe3a356609a3ee3457162ce7f8545 Mon Sep 17 00:00:00 2001 From: khyatisoneji <31898080+khyatisoneji@users.noreply.github.com> Date: Tue, 18 Feb 2020 17:01:37 +0530 Subject: [PATCH 247/257] store: add consistency delay to fetch blocks (#2009) Signed-off-by: khyatisoneji --- cmd/thanos/compact.go | 37 +--- cmd/thanos/store.go | 9 +- docs/components/store.md | 15 +- pkg/block/fetcher.go | 78 +++++++- pkg/block/fetcher_test.go | 296 ++++++++++++++++++++++++----- pkg/compact/compact.go | 108 ++--------- pkg/compact/compact_e2e_test.go | 72 ++----- pkg/testutil/e2eutil/prometheus.go | 128 +++++++++++++ test/e2e/store_gateway_test.go | 7 +- 9 files changed, 500 insertions(+), 250 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 4f4b9a578f..b0ea2f346c 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -16,7 +16,6 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" - "github.com/oklog/ulid" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -184,17 +183,11 @@ func runCompact( Name: "thanos_compactor_iterations_total", Help: "Total number of iterations that were executed successfully.", }) - consistencyDelayMetric := prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "thanos_consistency_delay_seconds", - Help: "Configured consistency delay in seconds.", - }, func() float64 { - return consistencyDelay.Seconds() - }) partialUploadDeleteAttempts := prometheus.NewCounter(prometheus.CounterOpts{ Name: "thanos_compactor_aborted_partial_uploads_deletion_attempts_total", Help: "Total number of started deletions of blocks that are assumed aborted and only partially uploaded.", }) - reg.MustRegister(halted, retried, iterations, consistencyDelayMetric, partialUploadDeleteAttempts) + reg.MustRegister(halted, retried, iterations, partialUploadDeleteAttempts) downsampleMetrics := newDownsampleMetrics(reg) @@ -247,15 +240,18 @@ func runCompact( } }() - metaFetcher, err := block.NewMetaFetcher(logger, 32, bkt, "", extprom.WrapRegistererWithPrefix("thanos_", reg), + duplicateBlocksFilter := block.NewDeduplicateFilter() + prometheusRegisterer := extprom.WrapRegistererWithPrefix("thanos_", reg) + metaFetcher, err := block.NewMetaFetcher(logger, 32, bkt, "", prometheusRegisterer, block.NewLabelShardedMetaFilter(relabelConfig).Filter, - (&consistencyDelayMetaFilter{logger: logger, consistencyDelay: consistencyDelay}).Filter, + block.NewConsistencyDelayMetaFilter(logger, consistencyDelay, prometheusRegisterer).Filter, + duplicateBlocksFilter.Filter, ) if err != nil { return errors.Wrap(err, "create meta fetcher") } - sy, err := compact.NewSyncer(logger, reg, bkt, metaFetcher, blockSyncConcurrency, acceptMalformedIndex, false) + sy, err := compact.NewSyncer(logger, reg, bkt, metaFetcher, duplicateBlocksFilter, blockSyncConcurrency, acceptMalformedIndex, false) if err != nil { return errors.Wrap(err, "create syncer") } @@ -392,25 +388,6 @@ func runCompact( return nil } -type consistencyDelayMetaFilter struct { - logger log.Logger - consistencyDelay time.Duration -} - -func (f *consistencyDelayMetaFilter) Filter(metas map[ulid.ULID]*metadata.Meta, synced block.GaugeLabeled, _ bool) { - for id, meta := range metas { - if ulid.Now()-id.Time() < uint64(f.consistencyDelay/time.Millisecond) && - meta.Thanos.Source != metadata.BucketRepairSource && - meta.Thanos.Source != metadata.CompactorSource && - meta.Thanos.Source != metadata.CompactorRepairSource { - - level.Debug(f.logger).Log("msg", "block is too fresh for now", "block", id) - synced.WithLabelValues(block.TooFreshMeta).Inc() - delete(metas, id) - } - } -} - // genMissingIndexCacheFiles scans over all blocks, generates missing index cache files and uploads them to object storage. func genMissingIndexCacheFiles(ctx context.Context, logger log.Logger, reg *prometheus.Registry, bkt objstore.Bucket, fetcher block.MetadataFetcher, dir string) error { genIndex := prometheus.NewCounter(prometheus.CounterOpts{ diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index e85f80e67d..4a25391484 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -81,6 +81,9 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { enableIndexHeader := cmd.Flag("experimental.enable-index-header", "If true, Store Gateway will recreate index-header instead of index-cache.json for each block. This will replace index-cache.json permanently once it will be out of experimental stage."). Hidden().Default("false").Bool() + consistencyDelay := modelDuration(cmd.Flag("consistency-delay", "Minimum age of all blocks before they are being read."). + Default("30m")) + m[component.Store.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, debugLogging bool) error { if minTime.PrometheusTimestamp() > maxTime.PrometheusTimestamp() { return errors.Errorf("invalid argument: --min-time '%s' can't be greater than --max-time '%s'", @@ -116,6 +119,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { selectorRelabelConf, *advertiseCompatibilityLabel, *enableIndexHeader, + time.Duration(*consistencyDelay), ) } } @@ -148,6 +152,7 @@ func runStore( selectorRelabelConf *extflag.PathOrContent, advertiseCompatibilityLabel bool, enableIndexHeader bool, + consistencyDelay time.Duration, ) error { grpcProbe := prober.NewGRPC() httpProbe := prober.NewHTTP() @@ -220,9 +225,11 @@ func runStore( return errors.Wrap(err, "create index cache") } - metaFetcher, err := block.NewMetaFetcher(logger, fetcherConcurrency, bkt, dataDir, extprom.WrapRegistererWithPrefix("thanos_", reg), + prometheusRegisterer := extprom.WrapRegistererWithPrefix("thanos_", reg) + metaFetcher, err := block.NewMetaFetcher(logger, fetcherConcurrency, bkt, dataDir, prometheusRegisterer, block.NewTimePartitionMetaFilter(filterConf.MinTime, filterConf.MaxTime).Filter, block.NewLabelShardedMetaFilter(relabelConfig).Filter, + block.NewConsistencyDelayMetaFilter(logger, consistencyDelay, prometheusRegisterer).Filter, block.NewDeduplicateFilter().Filter, ) if err != nil { diff --git a/docs/components/store.md b/docs/components/store.md index 3bda0a7351..f1e900212e 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -27,7 +27,8 @@ In general about 1MB of local disk space is required per TSDB block stored in th ## Flags -[embedmd]:# (flags/store.txt $) +[embedmd]: # "flags/store.txt $" + ```$ usage: thanos store [] @@ -137,6 +138,7 @@ Flags: Prometheus relabel-config syntax. See format details: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + --consistency-delay=30m Minimum age of all blocks before they are being read. ``` @@ -179,7 +181,8 @@ The `in-memory` index cache is enabled by default and its max size can be config Alternatively, the `in-memory` index cache can also by configured using `--index-cache.config-file` to reference to the configuration file or `--index-cache.config` to put yaml config directly: -[embedmd]:# (../flags/config_index_cache_in_memory.txt yaml) +[embedmd]: # "../flags/config_index_cache_in_memory.txt yaml" + ```yaml type: IN-MEMORY config: @@ -196,7 +199,8 @@ All the settings are **optional**: The `memcached` index cache allows to use [Memcached](https://memcached.org) as cache backend. This cache type is configured using `--index-cache.config-file` to reference to the configuration file or `--index-cache.config` to put yaml config directly: -[embedmd]:# (../flags/config_index_cache_memcached.txt yaml) +[embedmd]: # "../flags/config_index_cache_memcached.txt yaml" + ```yaml type: MEMCACHED config: @@ -224,13 +228,12 @@ While the remaining settings are **optional**: - `max_get_multi_batch_size`: maximum number of keys a single underlying operation should fetch. If more keys are specified, internally keys are splitted into multiple batches and fetched concurrently, honoring `max_get_multi_concurrency`. If set to `0`, the batch size is unlimited. - `dns_provider_update_interval`: the DNS discovery update interval. - ## Index Header In order to query series inside blocks from object storage, Store Gateway has to know certain initial info about each block such as: -* symbols table to unintern string values -* postings offset for posting lookup +- symbols table to unintern string values +- postings offset for posting lookup In order to achieve so, on startup for each block `index-header` is built from pieces of original block's index and stored on disk. Such `index-header` file is then mmaped and used by Store Gateway. diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index b6411e5431..d37dca38d0 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -417,7 +417,7 @@ func (f *LabelShardedMetaFilter) Filter(metas map[ulid.ULID]*metadata.Meta, sync // DeduplicateFilter is a MetaFetcher filter that filters out older blocks that have exactly the same data. type DeduplicateFilter struct { - DuplicateIDs []ulid.ULID + duplicateIDs []ulid.ULID } // NewDeduplicateFilter creates DeduplicateFilter. @@ -428,16 +428,30 @@ func NewDeduplicateFilter() *DeduplicateFilter { // Filter filters out duplicate blocks that can be formed // from two or more overlapping blocks that fully submatches the source blocks of the older blocks. func (f *DeduplicateFilter) Filter(metas map[ulid.ULID]*metadata.Meta, synced GaugeLabeled, _ bool) { - root := NewNode(&metadata.Meta{ - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(uint64(0), nil), - }, - }) + var wg sync.WaitGroup - metaSlice := []*metadata.Meta{} + metasByResolution := make(map[int64][]*metadata.Meta) for _, meta := range metas { - metaSlice = append(metaSlice, meta) + res := meta.Thanos.Downsample.Resolution + metasByResolution[res] = append(metasByResolution[res], meta) + } + + for res := range metasByResolution { + wg.Add(1) + go func(res int64) { + defer wg.Done() + f.filterForResolution(NewNode(&metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ + ULID: ulid.MustNew(uint64(0), nil), + }, + }), metasByResolution[res], metas, res, synced) + }(res) } + + wg.Wait() +} + +func (f *DeduplicateFilter) filterForResolution(root *Node, metaSlice []*metadata.Meta, metas map[ulid.ULID]*metadata.Meta, res int64, synced GaugeLabeled) { sort.Slice(metaSlice, func(i, j int) bool { ilen := len(metaSlice[i].Compaction.Sources) jlen := len(metaSlice[j].Compaction.Sources) @@ -456,13 +470,19 @@ func (f *DeduplicateFilter) Filter(metas map[ulid.ULID]*metadata.Meta, synced Ga duplicateULIDs := getNonRootIDs(root) for _, id := range duplicateULIDs { if metas[id] != nil { - f.DuplicateIDs = append(f.DuplicateIDs, id) + f.duplicateIDs = append(f.duplicateIDs, id) } synced.WithLabelValues(duplicateMeta).Inc() delete(metas, id) } } +// DuplicateIDs returns slice of block ids +// that are filtered out by DeduplicateFilter. +func (f *DeduplicateFilter) DuplicateIDs() []ulid.ULID { + return f.duplicateIDs +} + func addNodeBySources(root *Node, add *Node) bool { var rootNode *Node for _, node := range root.Children { @@ -506,3 +526,43 @@ func contains(s1 []ulid.ULID, s2 []ulid.ULID) bool { } return true } + +// ConsistencyDelayMetaFilter is a MetaFetcher filter that filters out blocks that are created before a specified consistency delay. +type ConsistencyDelayMetaFilter struct { + logger log.Logger + consistencyDelay time.Duration +} + +// NewConsistencyDelayMetaFilter creates ConsistencyDelayMetaFilter. +func NewConsistencyDelayMetaFilter(logger log.Logger, consistencyDelay time.Duration, reg prometheus.Registerer) *ConsistencyDelayMetaFilter { + consistencyDelayMetric := prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "consistency_delay_seconds", + Help: "Configured consistency delay in seconds.", + }, func() float64 { + return consistencyDelay.Seconds() + }) + reg.MustRegister(consistencyDelayMetric) + + return &ConsistencyDelayMetaFilter{ + logger: logger, + consistencyDelay: consistencyDelay, + } +} + +// Filter filters out blocks that filters blocks that have are created before a specified consistency delay. +func (f *ConsistencyDelayMetaFilter) Filter(metas map[ulid.ULID]*metadata.Meta, synced GaugeLabeled, _ bool) { + for id, meta := range metas { + // TODO(khyatisoneji): Remove the checks about Thanos Source + // by implementing delete delay to fetch metas. + // TODO(bwplotka): Check consistency delay based on file upload / modification time instead of ULID. + if ulid.Now()-id.Time() < uint64(f.consistencyDelay/time.Millisecond) && + meta.Thanos.Source != metadata.BucketRepairSource && + meta.Thanos.Source != metadata.CompactorSource && + meta.Thanos.Source != metadata.CompactorRepairSource { + + level.Debug(f.logger).Log("msg", "block is too fresh for now", "block", id) + synced.WithLabelValues(TooFreshMeta).Inc() + delete(metas, id) + } + } +} diff --git a/pkg/block/fetcher_test.go b/pkg/block/fetcher_test.go index 1f0cfa341d..9435d9d73c 100644 --- a/pkg/block/fetcher_test.go +++ b/pkg/block/fetcher_test.go @@ -396,18 +396,32 @@ func TestTimePartitionMetaFilter_Filter(t *testing.T) { } +type sourcesAndResolution struct { + sources []ulid.ULID + resolution int64 +} + func TestDeduplicateFilter_Filter(t *testing.T) { for _, tcase := range []struct { name string - input map[ulid.ULID][]ulid.ULID + input map[ulid.ULID]*sourcesAndResolution expected []ulid.ULID }{ { name: "3 non compacted blocks in bucket", - input: map[ulid.ULID][]ulid.ULID{ - ULID(1): []ulid.ULID{ULID(1)}, - ULID(2): []ulid.ULID{ULID(2)}, - ULID(3): []ulid.ULID{ULID(3)}, + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(1): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 0, + }, + ULID(2): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(2)}, + resolution: 0, + }, + ULID(3): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(3)}, + resolution: 0, + }, }, expected: []ulid.ULID{ ULID(1), @@ -417,10 +431,19 @@ func TestDeduplicateFilter_Filter(t *testing.T) { }, { name: "compacted block with sources in bucket", - input: map[ulid.ULID][]ulid.ULID{ - ULID(6): []ulid.ULID{ULID(6)}, - ULID(4): []ulid.ULID{ULID(1), ULID(3), ULID(2)}, - ULID(5): []ulid.ULID{ULID(5)}, + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(6): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(6)}, + resolution: 0, + }, + ULID(4): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(3), ULID(2)}, + resolution: 0, + }, + ULID(5): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(5)}, + resolution: 0, + }, }, expected: []ulid.ULID{ ULID(4), @@ -430,11 +453,23 @@ func TestDeduplicateFilter_Filter(t *testing.T) { }, { name: "two compacted blocks with same sources", - input: map[ulid.ULID][]ulid.ULID{ - ULID(5): []ulid.ULID{ULID(5)}, - ULID(6): []ulid.ULID{ULID(6)}, - ULID(3): []ulid.ULID{ULID(1), ULID(2)}, - ULID(4): []ulid.ULID{ULID(1), ULID(2)}, + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(5): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(5)}, + resolution: 0, + }, + ULID(6): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(6)}, + resolution: 0, + }, + ULID(3): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(2)}, + resolution: 0, + }, + ULID(4): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(2)}, + resolution: 0, + }, }, expected: []ulid.ULID{ ULID(3), @@ -444,10 +479,19 @@ func TestDeduplicateFilter_Filter(t *testing.T) { }, { name: "two compacted blocks with overlapping sources", - input: map[ulid.ULID][]ulid.ULID{ - ULID(4): []ulid.ULID{ULID(1), ULID(2)}, - ULID(6): []ulid.ULID{ULID(6)}, - ULID(5): []ulid.ULID{ULID(1), ULID(3), ULID(2)}, + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(4): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(2)}, + resolution: 0, + }, + ULID(6): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(6)}, + resolution: 0, + }, + ULID(5): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(3), ULID(2)}, + resolution: 0, + }, }, expected: []ulid.ULID{ ULID(5), @@ -456,12 +500,27 @@ func TestDeduplicateFilter_Filter(t *testing.T) { }, { name: "3 non compacted blocks and compacted block of level 2 in bucket", - input: map[ulid.ULID][]ulid.ULID{ - ULID(6): []ulid.ULID{ULID(6)}, - ULID(1): []ulid.ULID{ULID(1)}, - ULID(2): []ulid.ULID{ULID(2)}, - ULID(3): []ulid.ULID{ULID(3)}, - ULID(4): []ulid.ULID{ULID(2), ULID(1), ULID(3)}, + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(6): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(6)}, + resolution: 0, + }, + ULID(1): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 0, + }, + ULID(2): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(2)}, + resolution: 0, + }, + ULID(3): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(3)}, + resolution: 0, + }, + ULID(4): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(2), ULID(1), ULID(3)}, + resolution: 0, + }, }, expected: []ulid.ULID{ ULID(4), @@ -470,13 +529,31 @@ func TestDeduplicateFilter_Filter(t *testing.T) { }, { name: "3 compacted blocks of level 2 and one compacted block of level 3 in bucket", - input: map[ulid.ULID][]ulid.ULID{ - ULID(10): []ulid.ULID{ULID(1), ULID(2), ULID(3)}, - ULID(11): []ulid.ULID{ULID(6), ULID(4), ULID(5)}, - ULID(14): []ulid.ULID{ULID(14)}, - ULID(1): []ulid.ULID{ULID(1)}, - ULID(13): []ulid.ULID{ULID(1), ULID(6), ULID(2), ULID(3), ULID(5), ULID(7), ULID(4), ULID(8), ULID(9)}, - ULID(12): []ulid.ULID{ULID(7), ULID(9), ULID(8)}, + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(10): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(2), ULID(3)}, + resolution: 0, + }, + ULID(11): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(6), ULID(4), ULID(5)}, + resolution: 0, + }, + ULID(14): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(14)}, + resolution: 0, + }, + ULID(1): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 0, + }, + ULID(13): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(6), ULID(2), ULID(3), ULID(5), ULID(7), ULID(4), ULID(8), ULID(9)}, + resolution: 0, + }, + ULID(12): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(7), ULID(9), ULID(8)}, + resolution: 0, + }, }, expected: []ulid.ULID{ ULID(14), @@ -485,12 +562,27 @@ func TestDeduplicateFilter_Filter(t *testing.T) { }, { name: "compacted blocks with overlapping sources", - input: map[ulid.ULID][]ulid.ULID{ - ULID(8): []ulid.ULID{ULID(1), ULID(3), ULID(2), ULID(4)}, - ULID(1): []ulid.ULID{ULID(1)}, - ULID(5): []ulid.ULID{ULID(1), ULID(2)}, - ULID(6): []ulid.ULID{ULID(1), ULID(3), ULID(2), ULID(4)}, - ULID(7): []ulid.ULID{ULID(3), ULID(1), ULID(2)}, + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(8): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(3), ULID(2), ULID(4)}, + resolution: 0, + }, + ULID(1): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 0, + }, + ULID(5): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(2)}, + resolution: 0, + }, + ULID(6): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(3), ULID(2), ULID(4)}, + resolution: 0, + }, + ULID(7): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(3), ULID(1), ULID(2)}, + resolution: 0, + }, }, expected: []ulid.ULID{ ULID(6), @@ -498,10 +590,19 @@ func TestDeduplicateFilter_Filter(t *testing.T) { }, { name: "compacted blocks of level 3 with overlapping sources of equal length", - input: map[ulid.ULID][]ulid.ULID{ - ULID(10): []ulid.ULID{ULID(1), ULID(2), ULID(6), ULID(7)}, - ULID(1): []ulid.ULID{ULID(1)}, - ULID(11): []ulid.ULID{ULID(6), ULID(8), ULID(1), ULID(2)}, + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(10): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(2), ULID(6), ULID(7)}, + resolution: 0, + }, + ULID(1): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 0, + }, + ULID(11): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(6), ULID(8), ULID(1), ULID(2)}, + resolution: 0, + }, }, expected: []ulid.ULID{ ULID(10), @@ -510,29 +611,128 @@ func TestDeduplicateFilter_Filter(t *testing.T) { }, { name: "compacted blocks of level 3 with overlapping sources of different length", - input: map[ulid.ULID][]ulid.ULID{ - ULID(10): []ulid.ULID{ULID(6), ULID(7), ULID(1), ULID(2)}, - ULID(1): []ulid.ULID{ULID(1)}, - ULID(5): []ulid.ULID{ULID(1), ULID(2)}, - ULID(11): []ulid.ULID{ULID(2), ULID(3), ULID(1)}, + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(10): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(6), ULID(7), ULID(1), ULID(2)}, + resolution: 0, + }, + ULID(1): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 0, + }, + ULID(5): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(2)}, + resolution: 0, + }, + ULID(11): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(2), ULID(3), ULID(1)}, + resolution: 0, + }, }, expected: []ulid.ULID{ ULID(10), ULID(11), }, }, + { + name: "blocks with same sources and different resolutions", + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(1): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 0, + }, + ULID(2): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 1000, + }, + ULID(3): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 10000, + }, + }, + expected: []ulid.ULID{ + ULID(1), + ULID(2), + ULID(3), + }, + }, + { + name: "compacted blocks with overlapping sources and different resolutions", + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(1): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 0, + }, + ULID(6): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(6)}, + resolution: 10000, + }, + ULID(4): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(3), ULID(2)}, + resolution: 0, + }, + ULID(5): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(2), ULID(3), ULID(1)}, + resolution: 1000, + }, + }, + expected: []ulid.ULID{ + ULID(4), + ULID(5), + ULID(6), + }, + }, + { + name: "compacted blocks of level 3 with overlapping sources of different length and different resolutions", + input: map[ulid.ULID]*sourcesAndResolution{ + ULID(10): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(7), ULID(5), ULID(1), ULID(2)}, + resolution: 0, + }, + ULID(12): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(6), ULID(7), ULID(1)}, + resolution: 10000, + }, + ULID(1): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 0, + }, + ULID(13): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1)}, + resolution: 10000, + }, + ULID(5): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(1), ULID(2)}, + resolution: 0, + }, + ULID(11): &sourcesAndResolution{ + sources: []ulid.ULID{ULID(2), ULID(3), ULID(1)}, + resolution: 0, + }, + }, + expected: []ulid.ULID{ + ULID(10), + ULID(11), + ULID(12), + }, + }, } { f := NewDeduplicateFilter() if ok := t.Run(tcase.name, func(t *testing.T) { synced := prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"state"}) metas := make(map[ulid.ULID]*metadata.Meta) inputLen := len(tcase.input) - for id, sources := range tcase.input { + for id, metaInfo := range tcase.input { metas[id] = &metadata.Meta{ BlockMeta: tsdb.BlockMeta{ ULID: id, Compaction: tsdb.BlockMetaCompaction{ - Sources: sources, + Sources: metaInfo.sources, + }, + }, + Thanos: metadata.Thanos{ + Downsample: metadata.ThanosDownsample{ + Resolution: metaInfo.resolution, }, }, } diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 8c02f11c00..3a963783f8 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -49,6 +49,7 @@ type Syncer struct { metrics *syncerMetrics acceptMalformedIndex bool enableVerticalCompaction bool + duplicateBlocksFilter *block.DeduplicateFilter } type syncerMetrics struct { @@ -123,19 +124,20 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { // NewMetaSyncer returns a new Syncer for the given Bucket and directory. // Blocks must be at least as old as the sync delay for being considered. -func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, fetcher block.MetadataFetcher, blockSyncConcurrency int, acceptMalformedIndex bool, enableVerticalCompaction bool) (*Syncer, error) { +func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, fetcher block.MetadataFetcher, duplicateBlocksFilter *block.DeduplicateFilter, blockSyncConcurrency int, acceptMalformedIndex bool, enableVerticalCompaction bool) (*Syncer, error) { if logger == nil { logger = log.NewNopLogger() } return &Syncer{ - logger: logger, - reg: reg, - bkt: bkt, - fetcher: fetcher, - blocks: map[ulid.ULID]*metadata.Meta{}, - metrics: newSyncerMetrics(reg), - blockSyncConcurrency: blockSyncConcurrency, - acceptMalformedIndex: acceptMalformedIndex, + logger: logger, + reg: reg, + bkt: bkt, + fetcher: fetcher, + blocks: map[ulid.ULID]*metadata.Meta{}, + metrics: newSyncerMetrics(reg), + duplicateBlocksFilter: duplicateBlocksFilter, + blockSyncConcurrency: blockSyncConcurrency, + acceptMalformedIndex: acceptMalformedIndex, // The syncer offers an option to enable vertical compaction, even if it's // not currently used by Thanos, because the compactor is also used by Cortex // which needs vertical compaction. @@ -225,95 +227,14 @@ func (s *Syncer) Groups() (res []*Group, err error) { // GarbageCollect deletes blocks from the bucket if their data is available as part of a // block with a higher compaction level. +// Call to SyncMetas function is required to populate duplicateIDs in duplicateBlocksFilter. func (s *Syncer) GarbageCollect(ctx context.Context) error { s.mtx.Lock() defer s.mtx.Unlock() begin := time.Now() - // Run a separate round of garbage collections for each valid resolution. - for _, res := range []int64{ - downsample.ResLevel0, downsample.ResLevel1, downsample.ResLevel2, - } { - err := s.garbageCollect(ctx, res) - if err != nil { - s.metrics.garbageCollectionFailures.Inc() - } - s.metrics.garbageCollections.Inc() - s.metrics.garbageCollectionDuration.Observe(time.Since(begin).Seconds()) - - if err != nil { - return errors.Wrapf(err, "garbage collect resolution %d", res) - } - } - return nil -} - -func (s *Syncer) GarbageBlocks(resolution int64) (ids []ulid.ULID, err error) { - // Map each block to its highest priority parent. Initial blocks have themselves - // in their source section, i.e. are their own parent. - parents := map[ulid.ULID]ulid.ULID{} - - for id, meta := range s.blocks { - // Skip any block that has a different resolution. - if meta.Thanos.Downsample.Resolution != resolution { - continue - } - - // For each source block we contain, check whether we are the highest priority parent block. - for _, sid := range meta.Compaction.Sources { - pid, ok := parents[sid] - // No parents for the source block so far. - if !ok { - parents[sid] = id - continue - } - pmeta, ok := s.blocks[pid] - if !ok { - return nil, errors.Errorf("previous parent block %s not found", pid) - } - // The current block is the higher priority parent for the source if its - // compaction level is higher than that of the previously set parent. - // If compaction levels are equal, the more recent ULID wins. - // - // The ULID recency alone is not sufficient since races, e.g. induced - // by downtime of garbage collection, may re-compact blocks that are - // were already compacted into higher-level blocks multiple times. - level, plevel := meta.Compaction.Level, pmeta.Compaction.Level - - if level > plevel || (level == plevel && id.Compare(pid) > 0) { - parents[sid] = id - } - } - } - - // A block can safely be deleted if they are not the highest priority parent for - // any source block. - topParents := map[ulid.ULID]struct{}{} - for _, pid := range parents { - topParents[pid] = struct{}{} - } - - for id, meta := range s.blocks { - // Skip any block that has a different resolution. - if meta.Thanos.Downsample.Resolution != resolution { - continue - } - if _, ok := topParents[id]; ok { - continue - } - - ids = append(ids, id) - } - return ids, nil -} - -func (s *Syncer) garbageCollect(ctx context.Context, resolution int64) error { - garbageIds, err := s.GarbageBlocks(resolution) - if err != nil { - return err - } - + garbageIds := s.duplicateBlocksFilter.DuplicateIDs() for _, id := range garbageIds { if ctx.Err() != nil { return ctx.Err() @@ -327,6 +248,7 @@ func (s *Syncer) garbageCollect(ctx context.Context, resolution int64) error { err := block.Delete(delCtx, s.logger, s.bkt, id) cancel() if err != nil { + s.metrics.garbageCollectionFailures.Inc() return retry(errors.Wrapf(err, "delete block %s from bucket", id)) } @@ -335,6 +257,8 @@ func (s *Syncer) garbageCollect(ctx context.Context, resolution int64) error { delete(s.blocks, id) s.metrics.garbageCollectedBlocks.Inc() } + s.metrics.garbageCollections.Inc() + s.metrics.garbageCollectionDuration.Observe(time.Since(begin).Seconds()) return nil } diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 56869e5bdc..43ce297434 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "math/rand" "os" "path" "path/filepath" @@ -19,12 +18,10 @@ import ( "github.com/go-kit/kit/log" "github.com/oklog/ulid" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/objstore" @@ -91,10 +88,13 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) { testutil.Ok(t, bkt.Upload(ctx, path.Join(m.ULID.String(), metadata.MetaFilename), &buf)) } - metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil) + duplicateBlocksFilter := block.NewDeduplicateFilter() + metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil, + duplicateBlocksFilter.Filter, + ) testutil.Ok(t, err) - sy, err := NewSyncer(nil, nil, bkt, metaFetcher, 1, false, false) + sy, err := NewSyncer(nil, nil, bkt, metaFetcher, duplicateBlocksFilter, 1, false, false) testutil.Ok(t, err) // Do one initial synchronization with the bucket. @@ -164,10 +164,13 @@ func TestGroup_Compact_e2e(t *testing.T) { reg := prometheus.NewRegistry() - metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil) + duplicateBlocksFilter := block.NewDeduplicateFilter() + metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil, + duplicateBlocksFilter.Filter, + ) testutil.Ok(t, err) - sy, err := NewSyncer(nil, nil, bkt, metaFetcher, 5, false, false) + sy, err := NewSyncer(nil, nil, bkt, metaFetcher, duplicateBlocksFilter, 5, false, false) testutil.Ok(t, err) comp, err := tsdb.NewLeveledCompactor(ctx, reg, logger, []int64{1000, 3000}, nil) @@ -382,7 +385,7 @@ func createAndUpload(t testing.TB, bkt objstore.Bucket, blocks []blockgenSpec) ( var id ulid.ULID var err error if b.numSamples == 0 { - id, err = createEmptyBlock(prepareDir, b.mint, b.maxt, b.extLset, b.res) + id, err = e2eutil.CreateEmptyBlock(prepareDir, b.mint, b.maxt, b.extLset, b.res) } else { id, err = e2eutil.CreateBlock(ctx, prepareDir, b.series, b.numSamples, b.mint, b.maxt, b.extLset, b.res) } @@ -396,56 +399,3 @@ func createAndUpload(t testing.TB, bkt objstore.Bucket, blocks []blockgenSpec) ( } return metas } - -// createEmptyBlock produces empty block like it was the case before fix: https://github.com/prometheus/tsdb/pull/374. -// (Prometheus pre v2.7.0). -func createEmptyBlock(dir string, mint int64, maxt int64, extLset labels.Labels, resolution int64) (ulid.ULID, error) { - entropy := rand.New(rand.NewSource(time.Now().UnixNano())) - uid := ulid.MustNew(ulid.Now(), entropy) - - if err := os.Mkdir(path.Join(dir, uid.String()), os.ModePerm); err != nil { - return ulid.ULID{}, errors.Wrap(err, "close index") - } - - if err := os.Mkdir(path.Join(dir, uid.String(), "chunks"), os.ModePerm); err != nil { - return ulid.ULID{}, errors.Wrap(err, "close index") - } - - w, err := index.NewWriter(context.Background(), path.Join(dir, uid.String(), "index")) - if err != nil { - return ulid.ULID{}, errors.Wrap(err, "new index") - } - - if err := w.Close(); err != nil { - return ulid.ULID{}, errors.Wrap(err, "close index") - } - - m := tsdb.BlockMeta{ - Version: 1, - ULID: uid, - MinTime: mint, - MaxTime: maxt, - Compaction: tsdb.BlockMetaCompaction{ - Level: 1, - Sources: []ulid.ULID{uid}, - }, - } - b, err := json.Marshal(&m) - if err != nil { - return ulid.ULID{}, err - } - - if err := ioutil.WriteFile(path.Join(dir, uid.String(), "meta.json"), b, os.ModePerm); err != nil { - return ulid.ULID{}, errors.Wrap(err, "saving meta.json") - } - - if _, err = metadata.InjectThanos(log.NewNopLogger(), filepath.Join(dir, uid.String()), metadata.Thanos{ - Labels: extLset.Map(), - Downsample: metadata.ThanosDownsample{Resolution: resolution}, - Source: metadata.TestSource, - }, nil); err != nil { - return ulid.ULID{}, errors.Wrap(err, "finalize block") - } - - return uid, nil -} diff --git a/pkg/testutil/e2eutil/prometheus.go b/pkg/testutil/e2eutil/prometheus.go index b298ae91f9..fcc6ad2db5 100644 --- a/pkg/testutil/e2eutil/prometheus.go +++ b/pkg/testutil/e2eutil/prometheus.go @@ -5,6 +5,7 @@ package e2eutil import ( "context" + "encoding/json" "fmt" "io/ioutil" "math" @@ -12,6 +13,7 @@ import ( "net/http" "os" "os/exec" + "path" "path/filepath" "runtime" "strings" @@ -24,6 +26,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" @@ -277,6 +280,131 @@ func (p *Prometheus) Appender() tsdb.Appender { return p.db.Appender() } +// CreateEmptyBlock produces empty block like it was the case before fix: https://github.com/prometheus/tsdb/pull/374. +// (Prometheus pre v2.7.0). +func CreateEmptyBlock(dir string, mint int64, maxt int64, extLset labels.Labels, resolution int64) (ulid.ULID, error) { + entropy := rand.New(rand.NewSource(time.Now().UnixNano())) + uid := ulid.MustNew(ulid.Now(), entropy) + + if err := os.Mkdir(path.Join(dir, uid.String()), os.ModePerm); err != nil { + return ulid.ULID{}, errors.Wrap(err, "close index") + } + + if err := os.Mkdir(path.Join(dir, uid.String(), "chunks"), os.ModePerm); err != nil { + return ulid.ULID{}, errors.Wrap(err, "close index") + } + + w, err := index.NewWriter(context.Background(), path.Join(dir, uid.String(), "index")) + if err != nil { + return ulid.ULID{}, errors.Wrap(err, "new index") + } + + if err := w.Close(); err != nil { + return ulid.ULID{}, errors.Wrap(err, "close index") + } + + m := tsdb.BlockMeta{ + Version: 1, + ULID: uid, + MinTime: mint, + MaxTime: maxt, + Compaction: tsdb.BlockMetaCompaction{ + Level: 1, + Sources: []ulid.ULID{uid}, + }, + } + b, err := json.Marshal(&m) + if err != nil { + return ulid.ULID{}, err + } + + if err := ioutil.WriteFile(path.Join(dir, uid.String(), "meta.json"), b, os.ModePerm); err != nil { + return ulid.ULID{}, errors.Wrap(err, "saving meta.json") + } + + if _, err = metadata.InjectThanos(log.NewNopLogger(), filepath.Join(dir, uid.String()), metadata.Thanos{ + Labels: extLset.Map(), + Downsample: metadata.ThanosDownsample{Resolution: resolution}, + Source: metadata.TestSource, + }, nil); err != nil { + return ulid.ULID{}, errors.Wrap(err, "finalize block") + } + + return uid, nil +} + +// CreateBlockWithBlockDelay writes a block with the given series and numSamples samples each. +// Samples will be in the time range [mint, maxt) +// Block ID will be created with a delay of time duration blockDelay. +func CreateBlockWithBlockDelay( + ctx context.Context, + dir string, + series []labels.Labels, + numSamples int, + mint, maxt int64, + blockDelay time.Duration, + extLset labels.Labels, + resolution int64, +) (id ulid.ULID, err error) { + blockID, err := createBlock(ctx, dir, series, numSamples, mint, maxt, extLset, resolution, false) + if err != nil { + return id, errors.Wrap(err, "block creation") + } + + id, err = ulid.New(uint64(time.Unix(int64(blockID.Time()), 0).Add(-blockDelay*1000).Unix()), nil) + if err != nil { + return id, errors.Wrap(err, "create block id") + } + + if blockID.Compare(id) == 0 { + return + } + + metaFile := path.Join(dir, blockID.String(), "meta.json") + r, err := os.Open(metaFile) + if err != nil { + return id, errors.Wrap(err, "open meta file") + } + + metaContent, err := ioutil.ReadAll(r) + if err != nil { + return id, errors.Wrap(err, "read meta file") + } + + m := &metadata.Meta{} + if err := json.Unmarshal(metaContent, m); err != nil { + return id, errors.Wrap(err, "meta.json corrupted") + } + m.ULID = id + m.Compaction.Sources = []ulid.ULID{id} + + if err := os.MkdirAll(path.Join(dir, id.String()), 0777); err != nil { + return id, errors.Wrap(err, "create directory") + } + + err = copyRecursive(path.Join(dir, blockID.String()), path.Join(dir, id.String())) + if err != nil { + return id, errors.Wrap(err, "copy directory") + } + + err = os.RemoveAll(path.Join(dir, blockID.String())) + if err != nil { + return id, errors.Wrap(err, "delete directory") + } + + jsonMeta, err := json.MarshalIndent(m, "", "\t") + if err != nil { + return id, errors.Wrap(err, "meta marshal") + } + + err = ioutil.WriteFile(path.Join(dir, id.String(), "meta.json"), jsonMeta, 0644) + if err != nil { + return id, errors.Wrap(err, "write meta.json file") + } + + return +} + // CreateBlock writes a block with the given series and numSamples samples each. // Samples will be in the time range [mint, maxt). func CreateBlock( diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 484fbec325..72c8945480 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -75,18 +75,19 @@ func TestStoreGateway(t *testing.T) { series := []labels.Labels{ labels.FromStrings("a", "1", "b", "2"), } + extLset := labels.FromStrings("ext1", "value1", "replica", "1") extLset2 := labels.FromStrings("ext1", "value1", "replica", "2") extLset3 := labels.FromStrings("ext1", "value2", "replica", "3") now := time.Now() - id1, err := e2eutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset, 0) + id1, err := e2eutil.CreateBlockWithBlockDelay(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), 30*time.Minute, extLset, 0) testutil.Ok(t, err) - id2, err := e2eutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset2, 0) + id2, err := e2eutil.CreateBlockWithBlockDelay(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), 30*time.Minute, extLset2, 0) testutil.Ok(t, err) - id3, err := e2eutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset3, 0) + id3, err := e2eutil.CreateBlockWithBlockDelay(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), 30*time.Minute, extLset3, 0) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) From a354bfbac862ad6eab5af94d10f597079b6ad175 Mon Sep 17 00:00:00 2001 From: Aleksey Sin Date: Tue, 18 Feb 2020 18:22:45 +0300 Subject: [PATCH 248/257] query: Improve store response timeouts (#1789) * Improve proxyStore timeouts. Signed-off-by: Aleskey Sin * Fix send to closed channel. Signed-off-by: Aleskey Sin * Update for PR. Signed-off-by: Aleskey Sin * Fix recv done channel. Signed-off-by: Aleskey Sin * PR fixes. Signed-off-by: Aleskey Sin --- pkg/store/proxy.go | 122 ++++++------ pkg/store/proxy_test.go | 400 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 429 insertions(+), 93 deletions(-) diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 1af4242d28..c422f516f8 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -198,12 +198,7 @@ func newRespCh(ctx context.Context, buffer int) (*ctxRespSender, <-chan *storepb } func (s ctxRespSender) send(r *storepb.SeriesResponse) { - select { - case <-s.ctx.Done(): - return - case s.ch <- r: - return - } + s.ch <- r } // Series returns all series for a requested time range and label matcher. Requested series are taken from other @@ -348,6 +343,21 @@ type streamSeriesSet struct { closeSeries context.CancelFunc } +type recvResponse struct { + r *storepb.SeriesResponse + err error +} + +func frameCtx(responseTimeout time.Duration) (context.Context, context.CancelFunc) { + frameTimeoutCtx := context.Background() + var cancel context.CancelFunc + if responseTimeout != 0 { + frameTimeoutCtx, cancel = context.WithTimeout(frameTimeoutCtx, responseTimeout) + return frameTimeoutCtx, cancel + } + return frameTimeoutCtx, func() {} +} + func startStreamSeriesSet( ctx context.Context, logger log.Logger, @@ -383,78 +393,74 @@ func startStreamSeriesSet( emptyStreamResponses.Inc() } }() - for { - r, err := s.stream.Recv() - if err == io.EOF { - return - } - - if err != nil { - wrapErr := errors.Wrapf(err, "receive series from %s", s.name) - if partialResponse { - s.warnCh.send(storepb.NewWarnSeriesResponse(wrapErr)) + rCh := make(chan *recvResponse) + done := make(chan struct{}) + go func() { + for { + r, err := s.stream.Recv() + select { + case <-done: + close(rCh) return + case rCh <- &recvResponse{r: r, err: err}: } + } + }() + for { + frameTimeoutCtx, cancel := frameCtx(s.responseTimeout) + defer cancel() + var rr *recvResponse + select { + case <-ctx.Done(): + s.handleErr(errors.Wrapf(ctx.Err(), "failed to receive any data from %s", s.name), done) + return + case <-frameTimeoutCtx.Done(): + s.handleErr(errors.Wrapf(ctx.Err(), "failed to receive any data in %s from %s", s.responseTimeout.String(), s.name), done) + return + case rr = <-rCh: + } - s.errMtx.Lock() - s.err = wrapErr - s.errMtx.Unlock() + if rr.err == io.EOF { + close(done) return } + if rr.err != nil { + wrapErr := errors.Wrapf(rr.err, "receive series from %s", s.name) + s.handleErr(wrapErr, done) + return + } numResponses++ - if w := r.GetWarning(); w != "" { + if w := rr.r.GetWarning(); w != "" { s.warnCh.send(storepb.NewWarnSeriesResponse(errors.New(w))) continue } - - select { - case s.recvCh <- r.GetSeries(): - continue - case <-ctx.Done(): - return - } - + s.recvCh <- rr.r.GetSeries() } }() return s } -// Next blocks until new message is received or stream is closed or operation is timed out. -func (s *streamSeriesSet) Next() (ok bool) { - ctx := s.ctx - timeoutMsg := fmt.Sprintf("failed to receive any data from %s", s.name) - - if s.responseTimeout != 0 { - timeoutMsg = fmt.Sprintf("failed to receive any data in %s from %s", s.responseTimeout.String(), s.name) +func (s *streamSeriesSet) handleErr(err error, done chan struct{}) { + defer close(done) + s.closeSeries() - timeoutCtx, done := context.WithTimeout(s.ctx, s.responseTimeout) - defer done() - ctx = timeoutCtx + if s.partialResponse { + level.Warn(s.logger).Log("err", err, "msg", "returning partial response") + s.warnCh.send(storepb.NewWarnSeriesResponse(err)) + return } + s.errMtx.Lock() + s.err = err + s.errMtx.Unlock() +} - select { - case s.currSeries, ok = <-s.recvCh: - return ok - case <-ctx.Done(): - // closeSeries to shutdown a goroutine in startStreamSeriesSet. - s.closeSeries() - - err := errors.Wrap(ctx.Err(), timeoutMsg) - if s.partialResponse { - level.Warn(s.logger).Log("err", err, "msg", "returning partial response") - s.warnCh.send(storepb.NewWarnSeriesResponse(err)) - return false - } - s.errMtx.Lock() - s.err = err - s.errMtx.Unlock() - - level.Warn(s.logger).Log("err", err, "msg", "partial response disabled; aborting request") - return false - } +// Next blocks until new message is received or stream is closed or operation is timed out. +func (s *streamSeriesSet) Next() (ok bool) { + s.currSeries, ok = <-s.recvCh + return ok } func (s *streamSeriesSet) At() ([]storepb.Label, []storepb.AggrChunk) { diff --git a/pkg/store/proxy_test.go b/pkg/store/proxy_test.go index 0e5af9492f..cdc3ac7728 100644 --- a/pkg/store/proxy_test.go +++ b/pkg/store/proxy_test.go @@ -5,6 +5,7 @@ package store import ( "context" + "fmt" "io" "math" "os" @@ -49,6 +50,7 @@ func (c *testClient) String() string { func (c *testClient) Addr() string { return "testaddr" } + func TestProxyStore_Info(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() @@ -412,32 +414,6 @@ func TestProxyStore_Series(t *testing.T) { }, expectedErr: errors.New("fetch series for [name:\"ext\" value:\"1\" ] test: error!"), }, - { - title: "use no chunk to only get labels", - storeAPIs: []Client{ - &testClient{ - StoreClient: &mockedStoreAPI{ - RespSeries: []*storepb.SeriesResponse{ - storeSeriesResponse(t, labels.FromStrings("a", "a")), - }, - }, - minTime: 1, - maxTime: 300, - labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, - }, - }, - req: &storepb.SeriesRequest{ - MinTime: 1, - MaxTime: 300, - Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, - SkipChunks: true, - }, - expectedSeries: []rawSeries{ - { - lset: []storepb.Label{{Name: "a", Value: "a"}}, - }, - }, - }, } { if ok := t.Run(tc.title, func(t *testing.T) { @@ -488,8 +464,54 @@ func TestProxyStore_SeriesSlowStores(t *testing.T) { expectedWarningsLen int }{ { - title: "partial response disabled one thanos query is slow to respond", + title: "partial response disabled; 1st store is slow, 2nd store is fast;", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + RespDuration: 10 * time.Second, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + PartialResponseDisabled: true, + }, + expectedErr: errors.New("test: failed to receive any data in 4s from test: context deadline exceeded"), + }, + { + title: "partial response disabled; 1st store is fast, 2nd store is slow;", storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, &testClient{ StoreClient: &mockedStoreAPI{ RespSeries: []*storepb.SeriesResponse{ @@ -502,6 +524,33 @@ func TestProxyStore_SeriesSlowStores(t *testing.T) { minTime: 1, maxTime: 300, }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + PartialResponseDisabled: true, + }, + expectedErr: errors.New("test: failed to receive any data in 4s from test: context deadline exceeded"), + }, + { + title: "partial response disabled; 1st store is slow on 2nd series, 2nd store is fast;", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{3, 1}, {4, 2}, {5, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{6, 1}, {7, 2}, {8, 3}}), + }, + RespDuration: 10 * time.Second, + SlowSeriesIndex: 2, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, &testClient{ StoreClient: &mockedStoreAPI{ RespSeries: []*storepb.SeriesResponse{ @@ -523,7 +572,125 @@ func TestProxyStore_SeriesSlowStores(t *testing.T) { expectedErr: errors.New("test: failed to receive any data in 4s from test: context deadline exceeded"), }, { - title: "partial response enabled one thanos query is slow to respond", + title: "partial response disabled; 1st store is fast to respond, 2nd store is slow on 2nd series;", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{3, 1}, {4, 2}, {5, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{6, 1}, {7, 2}, {8, 3}}), + }, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + RespDuration: 10 * time.Second, + SlowSeriesIndex: 2, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + PartialResponseDisabled: true, + }, + expectedErr: errors.New("test: failed to receive any data in 4s from test: context deadline exceeded"), + }, + { + title: "partial response enabled; 1st store is slow to respond, 2nd store is fast;", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + RespDuration: 10 * time.Second, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("b", "c"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + }, + expectedSeries: []rawSeries{ + { + lset: []storepb.Label{{Name: "b", Value: "c"}}, + chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, + }, + }, + expectedWarningsLen: 2, + }, + { + title: "partial response enabled; 1st store is fast, 2nd store is slow;", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("b", "c"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + RespDuration: 10 * time.Second, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + }, + expectedSeries: []rawSeries{ + { + lset: []storepb.Label{{Name: "a", Value: "b"}}, + chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, + }, + }, + expectedWarningsLen: 2, + }, + { + title: "partial response enabled; 1st store is fast, 2-3 is slow, 4th is fast;", storeAPIs: []Client{ &testClient{ StoreClient: &mockedStoreAPI{ @@ -548,6 +715,154 @@ func TestProxyStore_SeriesSlowStores(t *testing.T) { minTime: 1, maxTime: 300, }, + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("c", "d"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + RespDuration: 10 * time.Second, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("d", "f"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + }, + expectedSeries: []rawSeries{ + { + lset: []storepb.Label{{Name: "a", Value: "b"}}, + chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, + }, + { + lset: []storepb.Label{{Name: "d", Value: "f"}}, + chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, + }, + }, + expectedWarningsLen: 4, + }, + { + title: "partial response enabled; 1st store is slow on 2nd series, 2nd store is fast", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{4, 1}, {5, 2}, {6, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{7, 1}, {8, 2}, {9, 3}}), + }, + RespDuration: 10 * time.Second, + SlowSeriesIndex: 2, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storepb.NewWarnSeriesResponse(errors.New("warning")), + storeSeriesResponse(t, labels.FromStrings("b", "c"), []sample{{1, 1}, {2, 2}, {3, 3}}), + }, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + }, + expectedSeries: []rawSeries{ + { + lset: []storepb.Label{{Name: "a", Value: "b"}}, + chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, + }, + { + lset: []storepb.Label{{Name: "b", Value: "c"}}, + chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, + }, + }, + expectedWarningsLen: 3, + }, + { + title: "partial response disabled; all stores respond 3s", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{4, 1}, {5, 2}, {6, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{7, 1}, {8, 2}, {9, 3}}), + }, + RespDuration: 3 * time.Second, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + PartialResponseDisabled: true, + }, + expectedSeries: []rawSeries{ + { + lset: []storepb.Label{{Name: "a", Value: "b"}}, + chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, + }, + }, + expectedErr: errors.New("test: failed to receive any data from test: context deadline exceeded"), + }, + { + title: "partial response enabled; all stores respond 3s", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{1, 1}, {2, 2}, {3, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{4, 1}, {5, 2}, {6, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{7, 1}, {8, 2}, {9, 3}}), + }, + RespDuration: 3 * time.Second, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storeSeriesResponse(t, labels.FromStrings("b", "c"), []sample{{1, 1}, {2, 2}, {3, 3}}), + storeSeriesResponse(t, labels.FromStrings("b", "c"), []sample{{4, 1}, {5, 2}, {6, 3}}), + storeSeriesResponse(t, labels.FromStrings("b", "c"), []sample{{7, 1}, {8, 2}, {9, 3}}), + }, + RespDuration: 3 * time.Second, + }, + labelSets: []storepb.LabelSet{{Labels: []storepb.Label{{Name: "ext", Value: "1"}}}}, + minTime: 1, + maxTime: 300, + }, }, req: &storepb.SeriesRequest{ MinTime: 1, @@ -559,6 +874,10 @@ func TestProxyStore_SeriesSlowStores(t *testing.T) { lset: []storepb.Label{{Name: "a", Value: "b"}}, chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, }, + { + lset: []storepb.Label{{Name: "b", Value: "c"}}, + chunks: [][]sample{{{1, 1}, {2, 2}, {3, 3}}}, + }, }, expectedWarningsLen: 2, }, @@ -572,9 +891,13 @@ func TestProxyStore_SeriesSlowStores(t *testing.T) { 4*time.Second, ) - s := newStoreSeriesServer(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + s := newStoreSeriesServer(ctx) + t0 := time.Now() err := q.Series(tc.req, s) + elapsedTime := time.Since(t0) if tc.expectedErr != nil { testutil.NotOk(t, err) testutil.Equals(t, tc.expectedErr.Error(), err.Error()) @@ -585,6 +908,8 @@ func TestProxyStore_SeriesSlowStores(t *testing.T) { seriesEquals(t, tc.expectedSeries, s.SeriesSet) testutil.Equals(t, tc.expectedWarningsLen, len(s.Warnings), "got %v", s.Warnings) + + testutil.Assert(t, elapsedTime < 5010*time.Millisecond, fmt.Sprintf("Request has taken %f, expected: <%d, it seems that responseTimeout doesn't work properly.", elapsedTime.Seconds(), 5)) }); !ok { return } @@ -1015,6 +1340,8 @@ type mockedStoreAPI struct { RespLabelNames *storepb.LabelNamesResponse RespError error RespDuration time.Duration + // Index of series in store to slow response. + SlowSeriesIndex int LastSeriesReq *storepb.SeriesRequest LastLabelValuesReq *storepb.LabelValuesRequest @@ -1028,7 +1355,7 @@ func (s *mockedStoreAPI) Info(ctx context.Context, req *storepb.InfoRequest, _ . func (s *mockedStoreAPI) Series(ctx context.Context, req *storepb.SeriesRequest, _ ...grpc.CallOption) (storepb.Store_SeriesClient, error) { s.LastSeriesReq = req - return &StoreSeriesClient{ctx: ctx, respSet: s.RespSeries, respDur: s.RespDuration}, s.RespError + return &StoreSeriesClient{ctx: ctx, respSet: s.RespSeries, respDur: s.RespDuration, slowSeriesIndex: s.SlowSeriesIndex}, s.RespError } func (s *mockedStoreAPI) LabelNames(ctx context.Context, req *storepb.LabelNamesRequest, _ ...grpc.CallOption) (*storepb.LabelNamesResponse, error) { @@ -1047,14 +1374,17 @@ func (s *mockedStoreAPI) LabelValues(ctx context.Context, req *storepb.LabelValu type StoreSeriesClient struct { // This field just exist to pseudo-implement the unused methods of the interface. storepb.Store_SeriesClient - ctx context.Context - i int - respSet []*storepb.SeriesResponse - respDur time.Duration + ctx context.Context + i int + respSet []*storepb.SeriesResponse + respDur time.Duration + slowSeriesIndex int } func (c *StoreSeriesClient) Recv() (*storepb.SeriesResponse, error) { - time.Sleep(c.respDur) + if c.respDur != 0 && (c.slowSeriesIndex == c.i || c.slowSeriesIndex == 0) { + time.Sleep(c.respDur) + } if c.i >= len(c.respSet) { return nil, io.EOF From 4a9671c0bc2df71d4d7b6c370340570e87b3b9da Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Wed, 19 Feb 2020 01:43:34 +0800 Subject: [PATCH 249/257] Makefile: update jsonnet-ci image (#2146) Signed-off-by: Xiang Dai <764524258@qq.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d26d2a88e2..c7c45219b3 100644 --- a/Makefile +++ b/Makefile @@ -317,7 +317,7 @@ JSONNET_CONTAINER_CMD:=docker run --rm \ -w "/go/src/github.com/thanos-io/thanos" \ -e USER=deadbeef \ -e GO111MODULE=on \ - quay.io/coreos/jsonnet-ci:release-0.35 + quay.io/coreos/jsonnet-ci:release-0.36 .PHONY: examples-in-container examples-in-container: From c39ddb2559ecaea4bcd754d8456016ed2e96f67e Mon Sep 17 00:00:00 2001 From: Aleksey Sin Date: Wed, 19 Feb 2020 14:09:03 +0300 Subject: [PATCH 250/257] Fix segfault panic. (#2153) Signed-off-by: Aleskey Sin --- pkg/store/proxy.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index c422f516f8..974a25215d 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -416,7 +416,7 @@ func startStreamSeriesSet( s.handleErr(errors.Wrapf(ctx.Err(), "failed to receive any data from %s", s.name), done) return case <-frameTimeoutCtx.Done(): - s.handleErr(errors.Wrapf(ctx.Err(), "failed to receive any data in %s from %s", s.responseTimeout.String(), s.name), done) + s.handleErr(errors.Wrapf(frameTimeoutCtx.Err(), "failed to receive any data in %s from %s", s.responseTimeout.String(), s.name), done) return case rr = <-rCh: } @@ -427,8 +427,7 @@ func startStreamSeriesSet( } if rr.err != nil { - wrapErr := errors.Wrapf(rr.err, "receive series from %s", s.name) - s.handleErr(wrapErr, done) + s.handleErr(errors.Wrapf(rr.err, "receive series from %s", s.name), done) return } numResponses++ From 49c566d44ba54074d8c352fa439445ef51cb91a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 19 Feb 2020 14:03:50 +0200 Subject: [PATCH 251/257] Add proposal for improving Thanos Query healthiness handling (#2086) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add proposal for improving Thanos Query healthiness handling Adds a document which proposes improving the healthiness handling of the store nodes in Thanos Query. Signed-off-by: Giedrius Statkevičius * docs/proposals: thanos_query_health_handling: update Update the proposal according to our discussion. Signed-off-by: Giedrius Statkevičius * docs: proposals: elaborate more on non-goals Elaborate more on what the 202001_thanos_query_health_handling.md proposal is not trying to solve and what problems will still exist after implementing this in terms of the end result caching. Signed-off-by: Giedrius Statkevičius --- .../202001_thanos_query_health_handling.md | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 docs/proposals/202001_thanos_query_health_handling.md diff --git a/docs/proposals/202001_thanos_query_health_handling.md b/docs/proposals/202001_thanos_query_health_handling.md new file mode 100644 index 0000000000..3782d4b6cb --- /dev/null +++ b/docs/proposals/202001_thanos_query_health_handling.md @@ -0,0 +1,96 @@ +--- +title: Thanos Query store nodes healthiness handling +type: proposal +menu: proposals +status: accepted +owner: GiedriusS +--- + +### Related Tickets + +* https://github.com/thanos-io/thanos/issues/1651 (Response caching for Thanos) +* https://github.com/thanos-io/thanos/pull/2072 (query: add 'sticky' nodes) +* https://github.com/cortexproject/cortex/pull/1974 (frontend: implement cache control) +* https://github.com/thanos-io/thanos/pull/1984 (api/v1: add cache-control header) + +## Summary + +This proposal document describes how currently the healthiness of store nodes is handled and why it should be changed (e.g. to improve response caching and avoid surprises). + +It explores a few options that are available - weighs their pros and cons, and finally proposes one final variant which we believe is the best out of the possible ones. + +## Motivation + +Currently Thanos Query updates the list of healthy store nodes every 5 seconds. It does that by sending the `Info()` call via gRPC. The last successful check is noted in the `LastCheck` field. At this point if it fails then we note the error and remove it from the set of active store nodes. If that succeeds then it becomes a part of the active store set. + +After `--store.unhealthy-timeout` passes since `LastCheck` then it also gets removed from the UI. At this point we would forget about it completely. + +Every time a query is executed, we consult the active store set and send the query to all of the nodes which match according to the external labels and the min/max times that they advertise. However, if a node goes down according to our previous definition then in 5 seconds we will not send anything to it anymore. This means that we won't even be able to control this via the partial response options since no query is sent to those nodes in the first place. + +This is problematic in the cache of end-response caching. If we know that certain StoreAPI nodes should always be up then we always want to have not just an error (if partial response is disabled) but also the appropriate `Cache-Control` header in the response. But right now we would only get it for maximum 5 seconds after a store node would go down. + +Thus, this logic needs to be changed somehow. There are a few possible options: + +1. `--store.unhealthy-timeout` could be made to apply to this case as well - we could still consider it a part of the active store set while it is still visible in the UI. +2. Another option could be introduced such as `--store.hold-timeout` which would be `--store.unhealthy-timeout`'s brother and we would hold the StoreAPI nodes for `max(hold_timeout, unhealthy_timeout)`. +3. Another option such as `--store.strict-mode` could be introduced which means that we would always retain the last information of the StoreAPI nodes of the last successful check. +4. The StoreAPI node specification format that is used in `--store` could be extended to include another flag which would let specify the previous option per-specific node. + +Lets look through their pros and cons: + +* In general it would be nice to avoid adding new options so the first option seems the most attractive in this regard but it is also the most invasive one. +* Second option increases the configuration complexity the most since you would have to think about the other option `--store.unhealthy-timeout` as well while setting it. +* The last option is the most complex from code's perspective but it is also least invasive; however it has some downsides like the syntax for specifying store nodes becomes really ugly and hard to understand because we would have not only the DNS query type in there but also a special marker to enable that mode. + +If we were to graph these choices in terms of their incisiveness and complexity it would look something like this: + +```text +Most incisive / Least Complex ------------ Least incisive / Most Complex +#1 #2 #4 + #3 +``` + +After careful consideration and with the rationale in this proposal, we have decided to go with the third option. It should provide a sweet spot between being too invasive and providing our users the ability to fall-back to the old behavior. + +## Goals + +* Update the health check logic to properly handle the case when a node disappears in terms of a caching layer. + +## No Goals + +* Fixing the cache handling in the cases where a **new** StoreAPI gets added to the active store set. + +This deserves a separate discussion and/or proposal. The issue when adding a completely new store node via service discovery is that a new node may suddenly provide new information in the past. In this paragraph when we are saying "new" it means new in terms of the data that it provides. Generally over time only a limited number of Prometheus instances will be providing data that can only change in the future (relatively to the current time). + +When this happens, we will need to most likely somehow signal the caching layer that it needs to drop (some of the) results cache that it has depending on: + +* Time ranges of a new node. +* Its external labels. +* The data that the node itself has by using some kind of hashing mechanism. + +The way this will need to be done should be as generic as possible so the design and solution of this is still an open question that this proposal does not solve. + +## Verification + +* Unit tests which would fire up a dummy StoreAPI and check different scenarios. +* Ad-hoc testing. + +## Proposal + +* Add a new flag to Thanos Query `--store.strict-mode` which will make it always retain the last successfully retrieved information via the `Info()` gRPC method of **statically** defined nodes and thus always consider them part of the active store set. + +## Risk + +* Users might have problems removing the store nodes from the active store set since they will be there forever with this option set. However, one might argue that if the nodes go down then something like DNS service discovery needs to be used which would dynamically add and remove those nodes. + +## Work Plan + +* Implement the new flag `--store.strict-mode` in Thanos Query which will make it keep around statically defined nodes. It will be disabled by default to reduce surprises when upgrading. +* Implement tests with dummy store nodes. +* Document the new behavior. + +## Future Work + +* Handle the case when a new node appears in terms of the end-result cache i.e. when using SD: + 1. Need to somehow signal the upper layer to clear the end-result cache. Ideally we would only clear the relevant parts. + 2. Perhaps some kind of hashing can be used for this. From 4061192d4a40f3ace7155d186e0d18c4b5634b59 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Wed, 19 Feb 2020 12:37:04 +0000 Subject: [PATCH 252/257] Fixed Segfault when matching large number of postings + LRU. (#2151) Fixes: https://github.com/thanos-io/thanos/issues/2147 Signed-off-by: Bartlomiej Plotka --- pkg/store/bucket_test.go | 177 ++++++++++++++++++++++++++++++++++++ pkg/store/cache/inmemory.go | 26 +++++- 2 files changed, 198 insertions(+), 5 deletions(-) diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 2fa3b2f089..142681b0e9 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -40,6 +40,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/filesystem" "github.com/thanos-io/thanos/pkg/objstore/inmem" "github.com/thanos-io/thanos/pkg/pool" + storecache "github.com/thanos-io/thanos/pkg/store/cache" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" "github.com/thanos-io/thanos/pkg/testutil/e2eutil" @@ -1278,3 +1279,179 @@ func benchmarkSeries(t testutil.TB, store *BucketStore, cases []*benchSeriesCase }) } } + +// Regression test against: https://github.com/thanos-io/thanos/issues/2147. +func TestSeries_OneBlock_InMemIndexCacheSegfault(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "segfault-series") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() + + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, bkt.Close()) }() + + logger := log.NewNopLogger() + + thanosMeta := metadata.Thanos{ + Labels: labels.Labels{{Name: "ext1", Value: "1"}}.Map(), + Downsample: metadata.ThanosDownsample{Resolution: 0}, + Source: metadata.TestSource, + } + + chunkPool, err := pool.NewBucketedBytesPool(maxChunkSize, 50e6, 2, 100e7) + testutil.Ok(t, err) + + indexCache, err := storecache.NewInMemoryIndexCacheWithConfig(logger, nil, storecache.InMemoryIndexCacheConfig{ + MaxItemSize: 3000, + // This is the exact size of cache needed for our *single request*. + // This is limited in order to make sure we test evictions. + MaxSize: 8889, + }) + testutil.Ok(t, err) + + var b1 *bucketBlock + + const numSeries = 100 + + // Create 4 blocks. Each will have numSeriesPerBlock number of series that have 1 sample only. + // Timestamp will be counted for each new series, so each series will have unique timestamp. + // This allows to pick time range that will correspond to number of series picked 1:1. + { + // Block 1. + h, err := tsdb.NewHead(nil, nil, nil, 1) + testutil.Ok(t, err) + defer testutil.Ok(t, h.Close()) + + app := h.Appender() + + for i := 0; i < numSeries; i++ { + ts := int64(i) + lbls := labels.FromStrings("foo", "bar", "b", "1", "i", fmt.Sprintf("%07d%s", ts, postingsBenchSuffix)) + + _, err := app.Add(lbls, ts, 0) + testutil.Ok(t, err) + } + testutil.Ok(t, app.Commit()) + + blockDir := filepath.Join(tmpDir, "tmp") + id := createBlockFromHead(t, blockDir, h) + + meta, err := metadata.InjectThanos(log.NewNopLogger(), filepath.Join(blockDir, id.String()), thanosMeta, nil) + testutil.Ok(t, err) + testutil.Ok(t, block.Upload(context.Background(), logger, bkt, filepath.Join(blockDir, id.String()))) + + b1 = &bucketBlock{ + indexCache: indexCache, + logger: logger, + bkt: bkt, + meta: meta, + partitioner: gapBasedPartitioner{maxGapSize: partitionerMaxGapSize}, + chunkObjs: []string{filepath.Join(id.String(), "chunks", "000001")}, + chunkPool: chunkPool, + } + b1.indexHeaderReader, err = indexheader.NewBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, b1.meta.ULID) + testutil.Ok(t, err) + } + + var b2 *bucketBlock + { + // Block 2, do not load this block yet. + h, err := tsdb.NewHead(nil, nil, nil, 1) + testutil.Ok(t, err) + defer testutil.Ok(t, h.Close()) + + app := h.Appender() + + for i := 0; i < numSeries; i++ { + ts := int64(i) + lbls := labels.FromStrings("foo", "bar", "b", "2", "i", fmt.Sprintf("%07d%s", ts, postingsBenchSuffix)) + + _, err := app.Add(lbls, ts, 0) + testutil.Ok(t, err) + } + testutil.Ok(t, app.Commit()) + + blockDir := filepath.Join(tmpDir, "tmp2") + id := createBlockFromHead(t, blockDir, h) + + meta, err := metadata.InjectThanos(log.NewNopLogger(), filepath.Join(blockDir, id.String()), thanosMeta, nil) + testutil.Ok(t, err) + testutil.Ok(t, block.Upload(context.Background(), logger, bkt, filepath.Join(blockDir, id.String()))) + + b2 = &bucketBlock{ + indexCache: indexCache, + logger: logger, + bkt: bkt, + meta: meta, + partitioner: gapBasedPartitioner{maxGapSize: partitionerMaxGapSize}, + chunkObjs: []string{filepath.Join(id.String(), "chunks", "000001")}, + chunkPool: chunkPool, + } + b2.indexHeaderReader, err = indexheader.NewBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, b2.meta.ULID) + testutil.Ok(t, err) + } + + store := &BucketStore{ + bkt: bkt, + logger: logger, + indexCache: indexCache, + metrics: newBucketStoreMetrics(nil), + blockSets: map[uint64]*bucketBlockSet{ + labels.Labels{{Name: "ext1", Value: "1"}}.Hash(): {blocks: [][]*bucketBlock{{b1, b2}}}, + }, + blocks: map[ulid.ULID]*bucketBlock{ + b1.meta.ULID: b1, + b2.meta.ULID: b2, + }, + queryGate: noopGater{}, + samplesLimiter: noopLimiter{}, + } + + t.Run("invoke series for one block. Fill the cache on the way.", func(t *testing.T) { + srv := newStoreSeriesServer(context.Background()) + testutil.Ok(t, store.Series(&storepb.SeriesRequest{ + MinTime: 0, + MaxTime: int64(numSeries) - 1, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "foo", Value: "bar"}, + {Type: storepb.LabelMatcher_EQ, Name: "b", Value: "1"}, + // This bug shows only when we use lot's of symbols for matching. + {Type: storepb.LabelMatcher_NEQ, Name: "i", Value: ""}, + }, + }, srv)) + testutil.Equals(t, 0, len(srv.Warnings)) + testutil.Equals(t, numSeries, len(srv.SeriesSet)) + }) + t.Run("invoke series for second block. This should revoke previous cache.", func(t *testing.T) { + srv := newStoreSeriesServer(context.Background()) + testutil.Ok(t, store.Series(&storepb.SeriesRequest{ + MinTime: 0, + MaxTime: int64(numSeries) - 1, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "foo", Value: "bar"}, + {Type: storepb.LabelMatcher_EQ, Name: "b", Value: "2"}, + // This bug shows only when we use lot's of symbols for matching. + {Type: storepb.LabelMatcher_NEQ, Name: "i", Value: ""}, + }, + }, srv)) + testutil.Equals(t, 0, len(srv.Warnings)) + testutil.Equals(t, numSeries, len(srv.SeriesSet)) + }) + t.Run("remove second block. Cache stays. Ask for first again.", func(t *testing.T) { + testutil.Ok(t, store.removeBlock(b2.meta.ULID)) + + srv := newStoreSeriesServer(context.Background()) + testutil.Ok(t, store.Series(&storepb.SeriesRequest{ + MinTime: 0, + MaxTime: int64(numSeries) - 1, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: "foo", Value: "bar"}, + {Type: storepb.LabelMatcher_EQ, Name: "b", Value: "1"}, + // This bug shows only when we use lot's of symbols for matching. + {Type: storepb.LabelMatcher_NEQ, Name: "i", Value: ""}, + }, + }, srv)) + testutil.Equals(t, 0, len(srv.Warnings)) + testutil.Equals(t, numSeries, len(srv.SeriesSet)) + }) +} diff --git a/pkg/store/cache/inmemory.go b/pkg/store/cache/inmemory.go index 02d5ad12ba..739d73b5bb 100644 --- a/pkg/store/cache/inmemory.go +++ b/pkg/store/cache/inmemory.go @@ -6,7 +6,9 @@ package storecache import ( "context" "math" + "reflect" "sync" + "unsafe" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -269,15 +271,29 @@ func (c *InMemoryIndexCache) reset() { c.curSize = 0 } +func copyString(s string) string { + var b []byte + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + h.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data + h.Len = len(s) + h.Cap = len(s) + return string(b) +} + +// copyToKey is required as underlying strings might be mmaped. +func copyToKey(l labels.Label) cacheKeyPostings { + return cacheKeyPostings(labels.Label{Value: copyString(l.Value), Name: copyString(l.Name)}) +} + // StorePostings sets the postings identified by the ulid and label to the value v, // if the postings already exists in the cache it is not mutated. -func (c *InMemoryIndexCache) StorePostings(ctx context.Context, blockID ulid.ULID, l labels.Label, v []byte) { - c.set(cacheTypePostings, cacheKey{blockID, cacheKeyPostings(l)}, v) +func (c *InMemoryIndexCache) StorePostings(_ context.Context, blockID ulid.ULID, l labels.Label, v []byte) { + c.set(cacheTypePostings, cacheKey{block: blockID, key: copyToKey(l)}, v) } // FetchMultiPostings fetches multiple postings - each identified by a label - // and returns a map containing cache hits, along with a list of missing keys. -func (c *InMemoryIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) { +func (c *InMemoryIndexCache) FetchMultiPostings(_ context.Context, blockID ulid.ULID, keys []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) { hits = map[labels.Label][]byte{} for _, key := range keys { @@ -294,13 +310,13 @@ func (c *InMemoryIndexCache) FetchMultiPostings(ctx context.Context, blockID uli // StoreSeries sets the series identified by the ulid and id to the value v, // if the series already exists in the cache it is not mutated. -func (c *InMemoryIndexCache) StoreSeries(ctx context.Context, blockID ulid.ULID, id uint64, v []byte) { +func (c *InMemoryIndexCache) StoreSeries(_ context.Context, blockID ulid.ULID, id uint64, v []byte) { c.set(cacheTypeSeries, cacheKey{blockID, cacheKeySeries(id)}, v) } // FetchMultiSeries fetches multiple series - each identified by ID - from the cache // and returns a map containing cache hits, along with a list of missing IDs. -func (c *InMemoryIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []uint64) (hits map[uint64][]byte, misses []uint64) { +func (c *InMemoryIndexCache) FetchMultiSeries(_ context.Context, blockID ulid.ULID, ids []uint64) (hits map[uint64][]byte, misses []uint64) { hits = map[uint64][]byte{} for _, id := range ids { From 021f623e7f3b29121f425ec2ce10e4e96377963a Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Thu, 20 Feb 2020 15:25:12 +0000 Subject: [PATCH 253/257] store: Added test for consistency delay filter, defaulted to 0. (#2159) Signed-off-by: Bartlomiej Plotka --- cmd/thanos/store.go | 4 +- pkg/block/fetcher.go | 9 ++-- pkg/block/fetcher_test.go | 109 +++++++++++++++++++++++++++++++++++++- pkg/extprom/testing.go | 35 ++++++++++++ 4 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 pkg/extprom/testing.go diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 4a25391484..90ecbce604 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -81,8 +81,8 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) { enableIndexHeader := cmd.Flag("experimental.enable-index-header", "If true, Store Gateway will recreate index-header instead of index-cache.json for each block. This will replace index-cache.json permanently once it will be out of experimental stage."). Hidden().Default("false").Bool() - consistencyDelay := modelDuration(cmd.Flag("consistency-delay", "Minimum age of all blocks before they are being read."). - Default("30m")) + consistencyDelay := modelDuration(cmd.Flag("consistency-delay", "Minimum age of all blocks before they are being read. Set it to safe value (e.g 30m) if your object storage is eventually consistent. GCS and S3 are (roughly) strongly consistent."). + Default("0s")) m[component.Store.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, debugLogging bool) error { if minTime.PrometheusTimestamp() > maxTime.PrometheusTimestamp() { diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index d37dca38d0..b68141531f 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -51,7 +51,7 @@ const ( // Filter's label values. labelExcludedMeta = "label-excluded" timeExcludedMeta = "time-excluded" - TooFreshMeta = "too-fresh" + tooFreshMeta = "too-fresh" duplicateMeta = "duplicate" ) @@ -83,7 +83,7 @@ func newSyncMetrics(r prometheus.Registerer) *syncMetrics { []string{corruptedMeta}, []string{noMeta}, []string{loadedMeta}, - []string{TooFreshMeta}, + []string{tooFreshMeta}, []string{failedMeta}, []string{labelExcludedMeta}, []string{timeExcludedMeta}, @@ -535,6 +535,9 @@ type ConsistencyDelayMetaFilter struct { // NewConsistencyDelayMetaFilter creates ConsistencyDelayMetaFilter. func NewConsistencyDelayMetaFilter(logger log.Logger, consistencyDelay time.Duration, reg prometheus.Registerer) *ConsistencyDelayMetaFilter { + if logger == nil { + logger = log.NewNopLogger() + } consistencyDelayMetric := prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "consistency_delay_seconds", Help: "Configured consistency delay in seconds.", @@ -561,7 +564,7 @@ func (f *ConsistencyDelayMetaFilter) Filter(metas map[ulid.ULID]*metadata.Meta, meta.Thanos.Source != metadata.CompactorRepairSource { level.Debug(f.logger).Log("msg", "block is too fresh for now", "block", id) - synced.WithLabelValues(TooFreshMeta).Inc() + synced.WithLabelValues(tooFreshMeta).Inc() delete(metas, id) } } diff --git a/pkg/block/fetcher_test.go b/pkg/block/fetcher_test.go index 9435d9d73c..7faf9d3062 100644 --- a/pkg/block/fetcher_test.go +++ b/pkg/block/fetcher_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math/rand" "os" "path" "path/filepath" @@ -25,6 +26,7 @@ import ( "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb" "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/objtesting" @@ -268,7 +270,7 @@ func TestMetaFetcher_Fetch(t *testing.T) { testutil.Equals(t, 0.0, promtest.ToFloat64(f.metrics.synced.WithLabelValues(labelExcludedMeta))) testutil.Equals(t, 0.0, promtest.ToFloat64(f.metrics.synced.WithLabelValues(timeExcludedMeta))) testutil.Equals(t, float64(expectedFailures), promtest.ToFloat64(f.metrics.synced.WithLabelValues(failedMeta))) - testutil.Equals(t, 0.0, promtest.ToFloat64(f.metrics.synced.WithLabelValues(TooFreshMeta))) + testutil.Equals(t, 0.0, promtest.ToFloat64(f.metrics.synced.WithLabelValues(tooFreshMeta))) }); !ok { return } @@ -770,3 +772,108 @@ func compareSliceWithMapKeys(tb testing.TB, m map[ulid.ULID]*metadata.Meta, s [] tb.FailNow() } } + +type ulidBuilder struct { + entropy *rand.Rand + + created []ulid.ULID +} + +func (u *ulidBuilder) ULID(t time.Time) ulid.ULID { + if u.entropy == nil { + source := rand.NewSource(1234) + u.entropy = rand.New(source) + } + + id := ulid.MustNew(ulid.Timestamp(t), u.entropy) + u.created = append(u.created, id) + return id +} + +func TestConsistencyDelayMetaFilter_Filter_0(t *testing.T) { + u := &ulidBuilder{} + now := time.Now() + + input := map[ulid.ULID]*metadata.Meta{ + // Fresh blocks. + u.ULID(now): {Thanos: metadata.Thanos{Source: metadata.SidecarSource}}, + u.ULID(now.Add(-1 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.SidecarSource}}, + u.ULID(now.Add(-1 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.ReceiveSource}}, + u.ULID(now.Add(-1 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.RulerSource}}, + + // For now non-delay delete sources, should be ignored by consistency delay. + u.ULID(now.Add(-1 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.BucketRepairSource}}, + u.ULID(now.Add(-1 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.CompactorSource}}, + u.ULID(now.Add(-1 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.CompactorRepairSource}}, + + // 29m. + u.ULID(now.Add(-29 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.SidecarSource}}, + u.ULID(now.Add(-29 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.ReceiveSource}}, + u.ULID(now.Add(-29 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.RulerSource}}, + + // For now non-delay delete sources, should be ignored by consistency delay. + u.ULID(now.Add(-29 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.BucketRepairSource}}, + u.ULID(now.Add(-29 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.CompactorSource}}, + u.ULID(now.Add(-29 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.CompactorRepairSource}}, + + // 30m. + u.ULID(now.Add(-30 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.SidecarSource}}, + u.ULID(now.Add(-30 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.ReceiveSource}}, + u.ULID(now.Add(-30 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.RulerSource}}, + u.ULID(now.Add(-30 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.BucketRepairSource}}, + u.ULID(now.Add(-30 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.CompactorSource}}, + u.ULID(now.Add(-30 * time.Minute)): {Thanos: metadata.Thanos{Source: metadata.CompactorRepairSource}}, + + // 30m+. + u.ULID(now.Add(-20 * time.Hour)): {Thanos: metadata.Thanos{Source: metadata.SidecarSource}}, + u.ULID(now.Add(-20 * time.Hour)): {Thanos: metadata.Thanos{Source: metadata.ReceiveSource}}, + u.ULID(now.Add(-20 * time.Hour)): {Thanos: metadata.Thanos{Source: metadata.RulerSource}}, + u.ULID(now.Add(-20 * time.Hour)): {Thanos: metadata.Thanos{Source: metadata.BucketRepairSource}}, + u.ULID(now.Add(-20 * time.Hour)): {Thanos: metadata.Thanos{Source: metadata.CompactorSource}}, + u.ULID(now.Add(-20 * time.Hour)): {Thanos: metadata.Thanos{Source: metadata.CompactorRepairSource}}, + } + + t.Run("consistency 0 (turned off)", func(t *testing.T) { + synced := prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"state"}) + expected := map[ulid.ULID]*metadata.Meta{} + // Copy all. + for _, id := range u.created { + expected[id] = input[id] + } + + reg := prometheus.NewRegistry() + f := NewConsistencyDelayMetaFilter(nil, 0*time.Second, reg) + testutil.Equals(t, map[string]float64{"consistency_delay_seconds": 0.0}, extprom.CurrentGaugeValuesFor(t, reg, "consistency_delay_seconds")) + + f.Filter(input, synced, false) + + testutil.Equals(t, 0.0, promtest.ToFloat64(synced.WithLabelValues(tooFreshMeta))) + testutil.Equals(t, expected, input) + }) + + t.Run("consistency 30m.", func(t *testing.T) { + synced := prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"state"}) + expected := map[ulid.ULID]*metadata.Meta{} + // Only certain sources and those with 30m or more age go through. + for i, id := range u.created { + // Younger than 30m. + if i < 13 { + if input[id].Thanos.Source != metadata.BucketRepairSource && + input[id].Thanos.Source != metadata.CompactorSource && + input[id].Thanos.Source != metadata.CompactorRepairSource { + continue + } + } + expected[id] = input[id] + } + + reg := prometheus.NewRegistry() + f := NewConsistencyDelayMetaFilter(nil, 30*time.Minute, reg) + testutil.Equals(t, map[string]float64{"consistency_delay_seconds": (30 * time.Minute).Seconds()}, extprom.CurrentGaugeValuesFor(t, reg, "consistency_delay_seconds")) + + f.Filter(input, synced, false) + + testutil.Equals(t, float64(len(u.created)-len(expected)), promtest.ToFloat64(synced.WithLabelValues(tooFreshMeta))) + testutil.Equals(t, expected, input) + }) +} diff --git a/pkg/extprom/testing.go b/pkg/extprom/testing.go new file mode 100644 index 0000000000..5db80b26e2 --- /dev/null +++ b/pkg/extprom/testing.go @@ -0,0 +1,35 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package extprom + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/thanos-io/thanos/pkg/testutil" +) + +// CurrentGaugeValuesFor returns gauge values for given metric names. Useful for testing based on registry, +// when you don't have access to metric variable. +func CurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames ...string) map[string]float64 { + f, err := reg.Gather() + testutil.Ok(t, err) + + res := make(map[string]float64, len(metricNames)) + for _, g := range f { + for _, m := range metricNames { + if g.GetName() != m { + continue + } + + testutil.Equals(t, 1, len(g.GetMetric())) + if _, ok := res[m]; ok { + t.Error("expected only one metric family for", m) + t.FailNow() + } + res[m] = *g.GetMetric()[0].GetGauge().Value + } + } + return res +} From dd60b8120c5ad935bbff004fe72bf4d5a56497c3 Mon Sep 17 00:00:00 2001 From: Matthias Loibl Date: Thu, 20 Feb 2020 18:06:18 +0100 Subject: [PATCH 254/257] Create release v0.11.0-rc.0 (#2156) * Update version to v0.11.0-rc.0 * Update CHANGELOG with all PRs for v0.11 * Improve CHANGELOG by being more explicit --- CHANGELOG.md | 26 ++++++++++++++----- VERSION | 2 +- .../thanos/1-globalview/courseBase.sh | 4 +-- .../katacoda/thanos/1-globalview/step2.md | 8 +++--- .../katacoda/thanos/1-globalview/step3.md | 2 +- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc68770465..3308263200 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,18 +11,29 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased +## [v0.11.0-rc.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0-rc.0) - 2020.02.19 + ### Fixed -- [#2033](https://github.com/thanos-io/thanos/pull/2033) minio-go: Fixed Issue #1494 support Web Identity providers for IAM credentials for AWS EKS -- [#1985](https://github.com/thanos-io/thanos/pull/1985) store gateway: Fixed case where series entry is larger than 64KB in index. -- [#2051](https://github.com/thanos-io/thanos/pull/2051) ruler: Fixed issue where ruler does not expose shipper metrics. -- [#2101](https://github.com/thanos-io/thanos/pull/2101) ruler: Fixed bug where thanos_alert_sender_errors_total was not registered. +- [#2033](https://github.com/thanos-io/thanos/pull/2033) Minio-go: Fixed Issue #1494 support Web Identity providers for IAM credentials for AWS EKS. +- [#1985](https://github.com/thanos-io/thanos/pull/1985) Store Gateway: Fixed case where series entry is larger than 64KB in index. +- [#2051](https://github.com/thanos-io/thanos/pull/2051) Ruler: Fixed issue where ruler does not expose shipper metrics. +- [#2101](https://github.com/thanos-io/thanos/pull/2101) Ruler: Fixed bug where thanos_alert_sender_errors_total was not registered. +- [#1789](https://github.com/thanos-io/thanos/pull/1789) Store Gateway: Improve timeouts. +- [#2139](https://github.com/thanos-io/thanos/pull/2139) Properly handle SIGHUP for reloading. +- [#2040](https://github.com/thanos-io/thanos/pull/2040) UI: Fix URL of alerts in Ruler +- [#2033](https://github.com/thanos-io/thanos/pull/1978) Ruler: Fix tracing in Thanos Ruler ### Added -- [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags -- [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction -- [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`. +- [#2003](https://github.com/thanos-io/thanos/pull/2003) Query: Support downsampling for /series. +- [#1952](https://github.com/thanos-io/thanos/pull/1952) Store Gateway: Implemented [binary index header](https://thanos.io/proposals/201912_thanos_binary_index_header.md/). This significantly reduces resource consumption (memory, CPU, net bandwidth) for startup and data loading processes as well as baseline memory. This means that adding more blocks into object storage, without querying them will use almost no resources. This, however, **still means that querying large amounts of data** will result in high spikes of memory and CPU use as before, due to simply fetching large amounts of metrics data. Since we fixed baseline, we are now focusing on query performance optimizations in separate initiatives. To enable experimental `index-header` mode run store with hidden `experimental.enable-index-header` flag. +- [#2009](https://github.com/thanos-io/thanos/pull/2009) Store Gateway: Minimum age of all blocks before they are being read. Set it to a safe value (e.g 30m) if your object storage is eventually consistent. GCS and S3 are (roughly) strongly consistent. +- [#1963](https://github.com/thanos-io/thanos/pull/1963) Mixin: Add Thanos Ruler alerts. +- [#1984](https://github.com/thanos-io/thanos/pull/1984) Query: Add cache-control header to not cache on error. +- [#1870](https://github.com/thanos-io/thanos/pull/1870) UI: Persist settings in query. +- [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags. +- [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction. - [#1939](https://github.com/thanos-io/thanos/pull/1939) Ruler: Add TLS and authentication support for query endpoints with the `--query.config` and `--query.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information. - [#1982](https://github.com/thanos-io/thanos/pull/1982) Ruler: Add support for Alertmanager v2 API endpoints. - [#2030](https://github.com/thanos-io/thanos/pull/2030) Query: Add `thanos_proxy_store_empty_stream_responses_total` metric for number of empty responses from stores. @@ -32,6 +43,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Changed +- [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`. - [#1933](https://github.com/thanos-io/thanos/pull/1933) Add a flag `--tsdb.wal-compression` to configure whether to enable tsdb wal compression in ruler and receiver. - [#2021](https://github.com/thanos-io/thanos/pull/2021) Rename metric `thanos_query_duplicated_store_address` to `thanos_query_duplicated_store_addresses_total` and `thanos_rule_duplicated_query_address` to `thanos_rule_duplicated_query_addresses_total`. diff --git a/VERSION b/VERSION index 539f9fc668..f395f4658b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.10.1-dev +0.11.0-rc.0 diff --git a/tutorials/katacoda/thanos/1-globalview/courseBase.sh b/tutorials/katacoda/thanos/1-globalview/courseBase.sh index 6bda9cb811..d9bcf83190 100644 --- a/tutorials/katacoda/thanos/1-globalview/courseBase.sh +++ b/tutorials/katacoda/thanos/1-globalview/courseBase.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash -docker pull quay.io/prometheus/prometheus:v2.14.0 -docker pull quay.io/thanos/thanos:v0.10.0 +docker pull quay.io/prometheus/prometheus:v2.16.0 +docker pull quay.io/thanos/thanos:0.11.0-rc.0 diff --git a/tutorials/katacoda/thanos/1-globalview/step2.md b/tutorials/katacoda/thanos/1-globalview/step2.md index 4982909dfe..bdaa76348e 100644 --- a/tutorials/katacoda/thanos/1-globalview/step2.md +++ b/tutorials/katacoda/thanos/1-globalview/step2.md @@ -10,7 +10,7 @@ component and can be invoked in a single command. Let's take a look at all the Thanos commands: ``` -docker run --rm quay.io/thanos/thanos:v0.10.0 --help +docker run --rm quay.io/thanos/thanos:0.11.0-rc.0 --help ```{{execute}} You should see multiple commands that solves different purposes. @@ -53,7 +53,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_eu1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-eu1 \ -u root \ - quay.io/thanos/thanos:v0.10.0 \ + quay.io/thanos/thanos:v0.11.0-rc.0 \ sidecar \ --http-address 0.0.0.0:19090 \ --grpc-address 0.0.0.0:19190 \ @@ -68,7 +68,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.10.0 \ + quay.io/thanos/thanos:v0.11.0-rc.0 \ sidecar \ --http-address 0.0.0.0:19091 \ --grpc-address 0.0.0.0:19191 \ @@ -81,7 +81,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus1_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-1-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.10.0 \ + quay.io/thanos/thanos:v0.11.0-rc.0 \ sidecar \ --http-address 0.0.0.0:19092 \ --grpc-address 0.0.0.0:19192 \ diff --git a/tutorials/katacoda/thanos/1-globalview/step3.md b/tutorials/katacoda/thanos/1-globalview/step3.md index cbe7d5f192..df0127cf7b 100644 --- a/tutorials/katacoda/thanos/1-globalview/step3.md +++ b/tutorials/katacoda/thanos/1-globalview/step3.md @@ -28,7 +28,7 @@ Click below snippet to start the Querier. ``` docker run -d --net=host --rm \ --name querier \ - quay.io/thanos/thanos:v0.10.0 \ + quay.io/thanos/thanos:v0.11.0-rc.0 \ query \ --http-address 0.0.0.0:29090 \ --query.replica-label replica \ From 7e2e832ed1a09c015ec3675eba7e55fada8d33b2 Mon Sep 17 00:00:00 2001 From: Kraig Amador <508403+bigkraig@users.noreply.github.com> Date: Tue, 25 Feb 2020 12:10:06 -0800 Subject: [PATCH 255/257] Bumped minio-go library to v6.0.49, fixing an IAM bug in v6.0.45 (#2189) Signed-off-by: Kraig Amador --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3308263200..44b5838771 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Fixed +- [#2189](https://github.com/thanos-io/thanos/pull/2189) minio-go: Fixed Issue #2181, unable to use IAM metadata credentials - [#2033](https://github.com/thanos-io/thanos/pull/2033) Minio-go: Fixed Issue #1494 support Web Identity providers for IAM credentials for AWS EKS. - [#1985](https://github.com/thanos-io/thanos/pull/1985) Store Gateway: Fixed case where series entry is larger than 64KB in index. - [#2051](https://github.com/thanos-io/thanos/pull/2051) Ruler: Fixed issue where ruler does not expose shipper metrics. diff --git a/go.mod b/go.mod index beb3c75d70..f9d1e93237 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe // indirect github.com/mattn/go-runewidth v0.0.6 // indirect github.com/miekg/dns v1.1.22 - github.com/minio/minio-go/v6 v6.0.45 + github.com/minio/minio-go/v6 v6.0.49 github.com/mozillazg/go-cos v0.13.0 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/oklog/run v1.0.0 diff --git a/go.sum b/go.sum index a4ddc0f20a..088b27019e 100644 --- a/go.sum +++ b/go.sum @@ -405,8 +405,8 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc= github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/minio/minio-go/v6 v6.0.45 h1:aY4NI/DOgSbZiwGN3fEF4NAkC9An4bhaIWuJrQrRYew= -github.com/minio/minio-go/v6 v6.0.45/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/minio-go/v6 v6.0.49 h1:bU4kIa/qChTLC1jrWZ8F+8gOiw1MClubddAJVR4gW3w= +github.com/minio/minio-go/v6 v6.0.49/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= From f223a1d17c3d6bfa14938ba155660fdcada618fb Mon Sep 17 00:00:00 2001 From: Matthias Loibl Date: Wed, 26 Feb 2020 13:48:12 +0100 Subject: [PATCH 256/257] Create release candidate v0.11.0-rc.1 (#2192) Signed-off-by: Matthias Loibl --- CHANGELOG.md | 2 +- VERSION | 2 +- tutorials/katacoda/thanos/1-globalview/courseBase.sh | 2 +- tutorials/katacoda/thanos/1-globalview/step2.md | 8 ++++---- tutorials/katacoda/thanos/1-globalview/step3.md | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44b5838771..ba9e811101 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased -## [v0.11.0-rc.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0-rc.0) - 2020.02.19 +## [v0.11.0-rc.1](https://github.com/thanos-io/thanos/releases/tag/v0.11.0-rc.1) - 2020.02.26 ### Fixed diff --git a/VERSION b/VERSION index f395f4658b..02526bda69 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.11.0-rc.0 +0.11.0-rc.1 diff --git a/tutorials/katacoda/thanos/1-globalview/courseBase.sh b/tutorials/katacoda/thanos/1-globalview/courseBase.sh index d9bcf83190..c71a57b774 100644 --- a/tutorials/katacoda/thanos/1-globalview/courseBase.sh +++ b/tutorials/katacoda/thanos/1-globalview/courseBase.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash docker pull quay.io/prometheus/prometheus:v2.16.0 -docker pull quay.io/thanos/thanos:0.11.0-rc.0 +docker pull quay.io/thanos/thanos:0.11.0-rc.1 diff --git a/tutorials/katacoda/thanos/1-globalview/step2.md b/tutorials/katacoda/thanos/1-globalview/step2.md index bdaa76348e..b766e924d2 100644 --- a/tutorials/katacoda/thanos/1-globalview/step2.md +++ b/tutorials/katacoda/thanos/1-globalview/step2.md @@ -10,7 +10,7 @@ component and can be invoked in a single command. Let's take a look at all the Thanos commands: ``` -docker run --rm quay.io/thanos/thanos:0.11.0-rc.0 --help +docker run --rm quay.io/thanos/thanos:0.11.0-rc.1 --help ```{{execute}} You should see multiple commands that solves different purposes. @@ -53,7 +53,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_eu1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-eu1 \ -u root \ - quay.io/thanos/thanos:v0.11.0-rc.0 \ + quay.io/thanos/thanos:v0.11.0-rc.1 \ sidecar \ --http-address 0.0.0.0:19090 \ --grpc-address 0.0.0.0:19190 \ @@ -68,7 +68,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.11.0-rc.0 \ + quay.io/thanos/thanos:v0.11.0-rc.1 \ sidecar \ --http-address 0.0.0.0:19091 \ --grpc-address 0.0.0.0:19191 \ @@ -81,7 +81,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus1_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-1-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.11.0-rc.0 \ + quay.io/thanos/thanos:v0.11.0-rc.1 \ sidecar \ --http-address 0.0.0.0:19092 \ --grpc-address 0.0.0.0:19192 \ diff --git a/tutorials/katacoda/thanos/1-globalview/step3.md b/tutorials/katacoda/thanos/1-globalview/step3.md index df0127cf7b..96f04a09a2 100644 --- a/tutorials/katacoda/thanos/1-globalview/step3.md +++ b/tutorials/katacoda/thanos/1-globalview/step3.md @@ -28,7 +28,7 @@ Click below snippet to start the Querier. ``` docker run -d --net=host --rm \ --name querier \ - quay.io/thanos/thanos:v0.11.0-rc.0 \ + quay.io/thanos/thanos:v0.11.0-rc.1 \ query \ --http-address 0.0.0.0:29090 \ --query.replica-label replica \ From 31cacc4930cc9c9ca9452fa149f89ee06248bd29 Mon Sep 17 00:00:00 2001 From: Matthias Loibl Date: Mon, 2 Mar 2020 12:32:10 +0100 Subject: [PATCH 257/257] Release v0.11.0 (#2205) Signed-off-by: Matthias Loibl --- CHANGELOG.md | 3 +-- VERSION | 2 +- tutorials/katacoda/thanos/1-globalview/courseBase.sh | 2 +- tutorials/katacoda/thanos/1-globalview/step2.md | 8 ++++---- tutorials/katacoda/thanos/1-globalview/step3.md | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba9e811101..195acaba96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,10 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased -## [v0.11.0-rc.1](https://github.com/thanos-io/thanos/releases/tag/v0.11.0-rc.1) - 2020.02.26 +## [v0.11.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0-rc.1) - 2020.03.02 ### Fixed -- [#2189](https://github.com/thanos-io/thanos/pull/2189) minio-go: Fixed Issue #2181, unable to use IAM metadata credentials - [#2033](https://github.com/thanos-io/thanos/pull/2033) Minio-go: Fixed Issue #1494 support Web Identity providers for IAM credentials for AWS EKS. - [#1985](https://github.com/thanos-io/thanos/pull/1985) Store Gateway: Fixed case where series entry is larger than 64KB in index. - [#2051](https://github.com/thanos-io/thanos/pull/2051) Ruler: Fixed issue where ruler does not expose shipper metrics. diff --git a/VERSION b/VERSION index 02526bda69..d9df1bbc0c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.11.0-rc.1 +0.11.0 diff --git a/tutorials/katacoda/thanos/1-globalview/courseBase.sh b/tutorials/katacoda/thanos/1-globalview/courseBase.sh index c71a57b774..71000d4519 100644 --- a/tutorials/katacoda/thanos/1-globalview/courseBase.sh +++ b/tutorials/katacoda/thanos/1-globalview/courseBase.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash docker pull quay.io/prometheus/prometheus:v2.16.0 -docker pull quay.io/thanos/thanos:0.11.0-rc.1 +docker pull quay.io/thanos/thanos:0.11.0 diff --git a/tutorials/katacoda/thanos/1-globalview/step2.md b/tutorials/katacoda/thanos/1-globalview/step2.md index b766e924d2..cbfb88b82a 100644 --- a/tutorials/katacoda/thanos/1-globalview/step2.md +++ b/tutorials/katacoda/thanos/1-globalview/step2.md @@ -10,7 +10,7 @@ component and can be invoked in a single command. Let's take a look at all the Thanos commands: ``` -docker run --rm quay.io/thanos/thanos:0.11.0-rc.1 --help +docker run --rm quay.io/thanos/thanos:0.11.0 --help ```{{execute}} You should see multiple commands that solves different purposes. @@ -53,7 +53,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_eu1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-eu1 \ -u root \ - quay.io/thanos/thanos:v0.11.0-rc.1 \ + quay.io/thanos/thanos:v0.11.0 \ sidecar \ --http-address 0.0.0.0:19090 \ --grpc-address 0.0.0.0:19190 \ @@ -68,7 +68,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus0_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-0-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.11.0-rc.1 \ + quay.io/thanos/thanos:v0.11.0 \ sidecar \ --http-address 0.0.0.0:19091 \ --grpc-address 0.0.0.0:19191 \ @@ -81,7 +81,7 @@ docker run -d --net=host --rm \ -v $(pwd)/prometheus1_us1.yml:/etc/prometheus/prometheus.yml \ --name prometheus-1-sidecar-us1 \ -u root \ - quay.io/thanos/thanos:v0.11.0-rc.1 \ + quay.io/thanos/thanos:v0.11.0 \ sidecar \ --http-address 0.0.0.0:19092 \ --grpc-address 0.0.0.0:19192 \ diff --git a/tutorials/katacoda/thanos/1-globalview/step3.md b/tutorials/katacoda/thanos/1-globalview/step3.md index 96f04a09a2..02d8931716 100644 --- a/tutorials/katacoda/thanos/1-globalview/step3.md +++ b/tutorials/katacoda/thanos/1-globalview/step3.md @@ -28,7 +28,7 @@ Click below snippet to start the Querier. ``` docker run -d --net=host --rm \ --name querier \ - quay.io/thanos/thanos:v0.11.0-rc.1 \ + quay.io/thanos/thanos:v0.11.0 \ query \ --http-address 0.0.0.0:29090 \ --query.replica-label replica \