Merge pull request #122 from kakkoyun/store_with_caching_bucket

store: Add mixin to enable caching bucket
thanos-io · May 20, 2020 · ad6e086 · ad6e086
2 parents 96ae241 + 128e08a
commit ad6e086
Show file tree

Hide file tree

Showing 30 changed files with 163 additions and 65 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,13 +17,17 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 - [#118](https://github.com/thanos-io/kube-thanos/pull/118) receive: Extend shutdown grace period to 900s (15min).
 
+- [#122](https://github.com/thanos-io/kube-thanos/pull/122) store: Rename `withMemcachedIndexCache` to `withIndexCacheMemcached`
+
 ### Added
 
 - [#105](https://github.com/thanos-io/kube-thanos/pull/105) compactor, store: Add deduplication replica label flags and delete delay labels
 
 - [#105](https://github.com/thanos-io/kube-thanos/pull/105) compactor, store: Add deduplication replica label flags and delete delay labels
 
-- [#119](https://github.com/thanos-io/kube-thanos/pull/119) receive: Distribute receive instances across node zones via pod anti affinity (note: only available on 1.17+).
+- [#119](https://github.com/thanos-io/kube-thanos/pull/119) receive: Distribute receive instances across node zones via pod anti affinity (note: only available on Kubernetes 1.17+)
+
+- [#122](https://github.com/thanos-io/kube-thanos/pull/122) store: Enable caching bucket support
 
 ### Fixed
 

diff --git a/README.md b/README.md
@@ -77,7 +77,7 @@ local commonConfig = {
   config+:: {
     local cfg = self,
     namespace: 'thanos',
-    version: 'v0.12.0',
+    version: 'v0.13.0-rc.0',
     image: 'quay.io/thanos/thanos:' + cfg.version,
     objectStorageConfig: {
       name: 'thanos-objectstorage',

diff --git a/all.jsonnet b/all.jsonnet
@@ -7,7 +7,7 @@ local commonConfig = {
   config+:: {
     local cfg = self,
     namespace: 'thanos',
-    version: 'v0.12.0',
+    version: 'v0.13.0-rc.0',
     image: 'quay.io/thanos/thanos:' + cfg.version,
     objectStorageConfig: {
       name: 'thanos-objectstorage',
@@ -83,7 +83,9 @@ local s =
 local swm =
   t.store +
   t.store.withVolumeClaimTemplate +
-  t.store.withServiceMonitor + t.store.withMemcachedIndexCache +
+  t.store.withServiceMonitor +
+  t.store.withIndexCacheMemcached +
+  t.store.withCachingBucketMemcached +
   commonConfig + {
     config+:: {
       name: 'thanos-store',

diff --git a/example.jsonnet b/example.jsonnet
@@ -7,7 +7,7 @@ local commonConfig = {
   config+:: {
     local cfg = self,
     namespace: 'thanos',
-    version: 'v0.12.0',
+    version: 'v0.13.0-rc.0',
     image: 'quay.io/thanos/thanos:' + cfg.version,
     objectStorageConfig: {
       name: 'thanos-objectstorage',

diff --git a/examples/all/manifests/thanos-bucket-deployment.yaml b/examples/all/manifests/thanos-bucket-deployment.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: object-store-bucket-debugging
     app.kubernetes.io/instance: thanos-bucket
     app.kubernetes.io/name: thanos-bucket
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-bucket
   namespace: thanos
 spec:
@@ -21,7 +21,7 @@ spec:
         app.kubernetes.io/component: object-store-bucket-debugging
         app.kubernetes.io/instance: thanos-bucket
         app.kubernetes.io/name: thanos-bucket
-        app.kubernetes.io/version: v0.12.0
+        app.kubernetes.io/version: v0.13.0-rc.0
     spec:
       containers:
       - args:
@@ -34,7 +34,7 @@ spec:
             secretKeyRef:
               key: thanos.yaml
               name: thanos-objectstorage
-        image: quay.io/thanos/thanos:v0.12.0
+        image: quay.io/thanos/thanos:v0.13.0-rc.0
         livenessProbe:
           failureThreshold: 4
           httpGet:

diff --git a/examples/all/manifests/thanos-bucket-service.yaml b/examples/all/manifests/thanos-bucket-service.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: object-store-bucket-debugging
     app.kubernetes.io/instance: thanos-bucket
     app.kubernetes.io/name: thanos-bucket
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-bucket
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-compact-service.yaml b/examples/all/manifests/thanos-compact-service.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: database-compactor
     app.kubernetes.io/instance: thanos-compact
     app.kubernetes.io/name: thanos-compact
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-compact
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-compact-serviceMonitor.yaml b/examples/all/manifests/thanos-compact-serviceMonitor.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: database-compactor
     app.kubernetes.io/instance: thanos-compact
     app.kubernetes.io/name: thanos-compact
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-compact
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-compact-statefulSet.yaml b/examples/all/manifests/thanos-compact-statefulSet.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: database-compactor
     app.kubernetes.io/instance: thanos-compact
     app.kubernetes.io/name: thanos-compact
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-compact
   namespace: thanos
 spec:
@@ -22,7 +22,7 @@ spec:
         app.kubernetes.io/component: database-compactor
         app.kubernetes.io/instance: thanos-compact
         app.kubernetes.io/name: thanos-compact
-        app.kubernetes.io/version: v0.12.0
+        app.kubernetes.io/version: v0.13.0-rc.0
     spec:
       containers:
       - args:
@@ -37,7 +37,7 @@ spec:
             secretKeyRef:
               key: thanos.yaml
               name: thanos-objectstorage
-        image: quay.io/thanos/thanos:v0.12.0
+        image: quay.io/thanos/thanos:v0.13.0-rc.0
         livenessProbe:
           failureThreshold: 4
           httpGet:

diff --git a/examples/all/manifests/thanos-query-deployment.yaml b/examples/all/manifests/thanos-query-deployment.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: query-layer
     app.kubernetes.io/instance: thanos-query
     app.kubernetes.io/name: thanos-query
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-query
   namespace: thanos
 spec:
@@ -21,7 +21,7 @@ spec:
         app.kubernetes.io/component: query-layer
         app.kubernetes.io/instance: thanos-query
         app.kubernetes.io/name: thanos-query
-        app.kubernetes.io/version: v0.12.0
+        app.kubernetes.io/version: v0.13.0-rc.0
     spec:
       affinity:
         podAntiAffinity:
@@ -47,7 +47,7 @@ spec:
         - --store=dnssrv+_grpc._tcp.thanos-receive.thanos.svc.cluster.local
         - --store=dnssrv+_grpc._tcp.thanos-rule.thanos.svc.cluster.local
         - --store=dnssrv+_grpc._tcp.thanos-store.thanos.svc.cluster.local
-        image: quay.io/thanos/thanos:v0.12.0
+        image: quay.io/thanos/thanos:v0.13.0-rc.0
         livenessProbe:
           failureThreshold: 4
           httpGet:

diff --git a/examples/all/manifests/thanos-query-service.yaml b/examples/all/manifests/thanos-query-service.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: query-layer
     app.kubernetes.io/instance: thanos-query
     app.kubernetes.io/name: thanos-query
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-query
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-query-serviceMonitor.yaml b/examples/all/manifests/thanos-query-serviceMonitor.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: query-layer
     app.kubernetes.io/instance: thanos-query
     app.kubernetes.io/name: thanos-query
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-query
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-receive-service.yaml b/examples/all/manifests/thanos-receive-service.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: database-write-hashring
     app.kubernetes.io/instance: thanos-receive
     app.kubernetes.io/name: thanos-receive
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-receive
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-receive-serviceMonitor.yaml b/examples/all/manifests/thanos-receive-serviceMonitor.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: database-write-hashring
     app.kubernetes.io/instance: thanos-receive
     app.kubernetes.io/name: thanos-receive
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-receive
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-receive-statefulSet.yaml b/examples/all/manifests/thanos-receive-statefulSet.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: database-write-hashring
     app.kubernetes.io/instance: thanos-receive
     app.kubernetes.io/name: thanos-receive
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-receive
   namespace: thanos
 spec:
@@ -22,7 +22,7 @@ spec:
         app.kubernetes.io/component: database-write-hashring
         app.kubernetes.io/instance: thanos-receive
         app.kubernetes.io/name: thanos-receive
-        app.kubernetes.io/version: v0.12.0
+        app.kubernetes.io/version: v0.13.0-rc.0
     spec:
       affinity:
         podAntiAffinity:
@@ -75,7 +75,7 @@ spec:
             secretKeyRef:
               key: thanos.yaml
               name: thanos-objectstorage
-        image: quay.io/thanos/thanos:v0.12.0
+        image: quay.io/thanos/thanos:v0.13.0-rc.0
         livenessProbe:
           failureThreshold: 8
           httpGet:

diff --git a/examples/all/manifests/thanos-rule-service.yaml b/examples/all/manifests/thanos-rule-service.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: rule-evaluation-engine
     app.kubernetes.io/instance: thanos-rule
     app.kubernetes.io/name: thanos-rule
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-rule
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-rule-serviceMonitor.yaml b/examples/all/manifests/thanos-rule-serviceMonitor.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: rule-evaluation-engine
     app.kubernetes.io/instance: thanos-rule
     app.kubernetes.io/name: thanos-rule
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-rule
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-rule-statefulSet.yaml b/examples/all/manifests/thanos-rule-statefulSet.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: rule-evaluation-engine
     app.kubernetes.io/instance: thanos-rule
     app.kubernetes.io/name: thanos-rule
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-rule
   namespace: thanos
 spec:
@@ -22,7 +22,7 @@ spec:
         app.kubernetes.io/component: rule-evaluation-engine
         app.kubernetes.io/instance: thanos-rule
         app.kubernetes.io/name: thanos-rule
-        app.kubernetes.io/version: v0.12.0
+        app.kubernetes.io/version: v0.13.0-rc.0
     spec:
       containers:
       - args:
@@ -44,7 +44,7 @@ spec:
             secretKeyRef:
               key: thanos.yaml
               name: thanos-objectstorage
-        image: quay.io/thanos/thanos:v0.12.0
+        image: quay.io/thanos/thanos:v0.13.0-rc.0
         livenessProbe:
           failureThreshold: 24
           httpGet:

diff --git a/examples/all/manifests/thanos-store-service.yaml b/examples/all/manifests/thanos-store-service.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: object-store-gateway
     app.kubernetes.io/instance: thanos-store
     app.kubernetes.io/name: thanos-store
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-store
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-store-serviceMonitor.yaml b/examples/all/manifests/thanos-store-serviceMonitor.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: object-store-gateway
     app.kubernetes.io/instance: thanos-store
     app.kubernetes.io/name: thanos-store
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-store
   namespace: thanos
 spec:

diff --git a/examples/all/manifests/thanos-store-statefulSet-with-memcached.yaml b/examples/all/manifests/thanos-store-statefulSet-with-memcached.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: object-store-gateway
     app.kubernetes.io/instance: thanos-store
     app.kubernetes.io/name: thanos-store
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-store
   namespace: thanos
 spec:
@@ -22,7 +22,7 @@ spec:
         app.kubernetes.io/component: object-store-gateway
         app.kubernetes.io/instance: thanos-store
         app.kubernetes.io/name: thanos-store
-        app.kubernetes.io/version: v0.12.0
+        app.kubernetes.io/version: v0.13.0-rc.0
     spec:
       containers:
       - args:
@@ -45,13 +45,35 @@ spec:
             "max_item_size": "1MiB"
             "timeout": "500ms"
           "type": "MEMCACHED"
+        - |-
+          --store.caching-bucket.config="blocks_iter_ttl": "5m"
+          "chunk_object_size_ttl": "24h"
+          "chunk_subrange_size": 16000
+          "chunk_subrange_ttl": "24h"
+          "config":
+            "addresses":
+            - "dnssrv+_client._tcp.<MEMCACHED_SERCIVE>.thanos.svc.cluster.local"
+            "dns_provider_update_interval": "10s"
+            "max_async_buffer_size": 10000
+            "max_async_concurrency": 20
+            "max_get_multi_batch_size": 0
+            "max_get_multi_concurrency": 100
+            "max_idle_connections": 100
+            "max_item_size": "1MiB"
+            "timeout": "500ms"
+          "max_chunks_get_range_requests": 3
+          "metafile_content_ttl": "24h"
+          "metafile_doesnt_exist_ttl": "15m"
+          "metafile_exists_ttl": "2h"
+          "metafile_max_size": "1MiB"
+          "type": "MEMCACHED"
         env:
         - name: OBJSTORE_CONFIG
           valueFrom:
             secretKeyRef:
               key: thanos.yaml
               name: thanos-objectstorage
-        image: quay.io/thanos/thanos:v0.12.0
+        image: quay.io/thanos/thanos:v0.13.0-rc.0
         livenessProbe:
           failureThreshold: 8
           httpGet:

diff --git a/examples/all/manifests/thanos-store-statefulSet.yaml b/examples/all/manifests/thanos-store-statefulSet.yaml
@@ -5,7 +5,7 @@ metadata:
     app.kubernetes.io/component: object-store-gateway
     app.kubernetes.io/instance: thanos-store
     app.kubernetes.io/name: thanos-store
-    app.kubernetes.io/version: v0.12.0
+    app.kubernetes.io/version: v0.13.0-rc.0
   name: thanos-store
   namespace: thanos
 spec:
@@ -22,7 +22,7 @@ spec:
         app.kubernetes.io/component: object-store-gateway
         app.kubernetes.io/instance: thanos-store
         app.kubernetes.io/name: thanos-store
-        app.kubernetes.io/version: v0.12.0
+        app.kubernetes.io/version: v0.13.0-rc.0
     spec:
       containers:
       - args:
@@ -37,7 +37,7 @@ spec:
             secretKeyRef:
               key: thanos.yaml
               name: thanos-objectstorage
-        image: quay.io/thanos/thanos:v0.12.0
+        image: quay.io/thanos/thanos:v0.13.0-rc.0
         livenessProbe:
           failureThreshold: 8
           httpGet:

diff --git a/examples/thanos-receive.jsonnet b/examples/thanos-receive.jsonnet
@@ -7,7 +7,7 @@ t.receive {
   local tr = self,
   name:: 'thanos-receive',
   namespace:: 'observability',
-  version:: 'v0.12.0-rc.0',
+  version:: 'v0.13.0-rc.0',
   image:: 'quay.io/thanos/thanos:v' + tr.version,
   replicas:: 3,
   replicationFactor:: 3,