openshift · simonpasquier · Oct 14, 2019 · Oct 14, 2019 · Oct 15, 2019 · Oct 15, 2019
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -8,7 +8,7 @@ jobs:
     working_directory: /go/src/github.com/thanos-io/thanos
     environment:
       GO111MODULE: 'on'
-      # Run garbage collection more aggresively to avoid getting OOMed during the lint phase (4GB limit).
+      # Run garbage collection more aggressively to avoid getting OOMed during the lint phase (4GB limit).
       GOGC: "20"
       # By default Go uses GOMAXPROCS but a Circle CI executor has many
       # cores (> 30) while the CPU and RAM resources are throttled. If we
@@ -26,22 +26,21 @@ jobs:
           command: |
             if ! [ -z ${GCP_PROJECT} ]; then
               echo $GOOGLE_APPLICATION_CREDENTIALS_CONTENT > $GOOGLE_APPLICATION_CREDENTIALS
-              echo "Awesome! GCS integration tests are enabled."
+              echo "Awesome! GCS and S3 AWS integration tests are enabled."
             fi
       - run: make deps
       - run: make lint
       - run: make check-docs
       - run: make format
       - run:
           name: "Run all tests"
-          # TODO(bwplotka): Setup some S3 tests for CI.
           # taskset sets CPU affinity to 2 (current CPU limit).
           command: |
             if [ -z ${GCP_PROJECT} ]; then
               taskset 2 make test-local
               exit
             fi
-            taskset 2 make test-only-gcs
+            taskset 2 make test-ci
 
   # Cross build is needed for publish_release but needs to be done outside of docker.
   cross_build:
@@ -60,7 +59,7 @@ jobs:
   publish_master:
     docker:
       # Build by Thanos make docker-ci
-      - image: quay.io/thanos/thanos-ci:v0.1.0
+      - image: quay.io/thanos/thanos-ci:v0.2.0
     working_directory: /go/src/github.com/thanos-io/thanos
     steps:
       - checkout
@@ -79,8 +78,8 @@ jobs:
 
   publish_release:
     docker:
-      # Available from https://hub.docker.com/r/circleci/golang/
-      - image: circleci/golang:1.13.1
+      # Build by Thanos make docker-ci
+      - image: quay.io/thanos/thanos-ci:v0.2.0
     working_directory: /go/src/github.com/thanos-io/thanos
     environment:
       GOBIN: "/go/bin"

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -3,12 +3,12 @@
     about what components it touches e.g "query:" or ".*:"
 -->
 
-<!-- 
-    Don't forget about CHANGELOG! 
-    
+<!--
+    Don't forget about CHANGELOG!
+
     Changelog entry format:
     - [#<PR-id>](<PR-URL>) Thanos <Component> ...
-    
+
     <PR-id> Id of your pull request.
     <PR-URL> URL of your PR such as https://github.com/thanos-io/thanos/pull/<PR-id>
     <Component> Component affected by your changes such as Query, Store, Receive.

diff --git a/.github/stale.yaml b/.github/stale.yaml
@@ -0,0 +1,16 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 90
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 14
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+# Label to use when marking an issue as stale
+staleLabel: stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions. Use `pinned` label to avoid closing.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -0,0 +1,21 @@
+name: CI
+on: [pull_request]
+
+jobs:
+  cross-build-check:
+    name: Cross build check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v1
+
+      - uses: actions/cache@v1
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+
+      - name: Get dependencies
+        run: make deps
+
+      - name: Cross build check
+        run: make crossbuild
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@
 
 /prometheus
 /thanos
+/mixin/vendor/
 
 # Ignore minikube setup working dirs.
 kube/bin
@@ -24,3 +25,4 @@ website/docs-pre-processed/
 !website/data
 
 tmp/bin
+examples/tmp/
diff --git a/.golangci.yml b/.golangci.yml
@@ -1,3 +1,47 @@
+# This file contains all available configuration options
+# with their default values.
+
+# options for analysis running
+run:
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  deadline: 5m
+
+  # exit code when at least one issue was found, default is 1
+  issues-exit-code: 1
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #     vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs: vendor
+
+# output configuration options
+output:
+  # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+linters:
+  enable:
+    - deadcode
+    - errcheck
+    - goconst
+    - goimports
+    - gosimple
+    - govet
+    - ineffassign
+    - staticcheck
+    - structcheck
+    - typecheck
+    - unused
+    - varcheck
+
 linters-settings:
     errcheck:
         exclude: ./.errcheck_excludes.txt

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,22 +9,103 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 
 We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.)
 
-## Unreleased
+## [v0.10.0](https://github.com/thanos-io/thanos/releases/tag/v0.10.0) - 2020.01.13
+
+### Fixed
+
+- [#1919](https://github.com/thanos-io/thanos/issues/1919) Compactor: Fixed potential data loss when uploading older blocks, or upload taking long time while compactor is
+running.
+- [#1937](https://github.com/thanos-io/thanos/pull/1937) Compactor: Improved synchronization of meta JSON files.
+Compactor now properly handles partial block uploads for all operation like retention apply, downsampling and compaction. Additionally:
+
+  * Removed `thanos_compact_sync_meta_*` metrics. Use `thanos_blocks_meta_*` metrics instead.
+  * Added `thanos_consistency_delay_seconds` and `thanos_compactor_aborted_partial_uploads_deletion_attempts_total` metrics.
+
+- [#1936](https://github.com/thanos-io/thanos/pull/1936) Store: Improved synchronization of meta JSON files. Store now properly handles corrupted disk cache. Added meta.json sync metrics.
+- [#1856](https://github.com/thanos-io/thanos/pull/1856) Receive: close DBReadOnly after flushing to fix a memory leak.
+- [#1882](https://github.com/thanos-io/thanos/pull/1882) Receive: upload to object storage as 'receive' rather than 'sidecar'.
+- [#1907](https://github.com/thanos-io/thanos/pull/1907) Store: Fixed the duration unit for the metric `thanos_bucket_store_series_gate_duration_seconds`.
+- [#1931](https://github.com/thanos-io/thanos/pull/1931) Compact: Fixed the compactor successfully exiting when actually an error occurred while compacting a blocks group.
+- [#1872](https://github.com/thanos-io/thanos/pull/1872) Ruler: `/api/v1/rules` now shows a properly formatted value
+- [#1945](https://github.com/thanos-io/thanos/pull/1945) `master` container images are now built with Go 1.13
+- [#1956](https://github.com/thanos-io/thanos/pull/1956) Ruler: now properly ignores duplicated query addresses
+- [#1975](https://github.com/thanos-io/thanos/pull/1975) Store Gateway: fixed panic caused by memcached servers selector when there's 1 memcached node
+
+### Added
+
+- [#1852](https://github.com/thanos-io/thanos/pull/1852) Add support for `AWS_CONTAINER_CREDENTIALS_FULL_URI` by upgrading to minio-go v6.0.44
+- [#1854](https://github.com/thanos-io/thanos/pull/1854) Update Rule UI to support alerts count displaying and filtering.
+- [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add TLS and authentication support for Alertmanager with the `--alertmanagers.config` and `--alertmanagers.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information.
+- [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add a new `--alertmanagers.sd-dns-interval` CLI option to specify the interval between DNS resolutions of Alertmanager hosts.
+- [#1881](https://github.com/thanos-io/thanos/pull/1881) Store Gateway: memcached support for index cache. See [documentation](docs/components/store.md/#index-cache) for further information.
+- [#1904](https://github.com/thanos-io/thanos/pull/1904) Add a skip-chunks option in Store Series API to improve the response time of `/api/v1/series` endpoint.
+- [#1910](https://github.com/thanos-io/thanos/pull/1910) Query: `/api/v1/labels` now understands `POST` - useful for sending bigger requests
+
+### Changed
+
+- [#1947](https://github.com/thanos-io/thanos/pull/1947) Upgraded Prometheus dependencies to v2.15.2. This includes:
+
+  * Compactor: Significant reduction of memory footprint for compaction and downsampling process.
+  * Querier: Accepting spaces between time range and square bracket. e.g `[ 5m]`
+  * Querier: Improved PromQL parser performance.
+
+- [#1833](https://github.com/thanos-io/thanos/pull/1833) `--shipper.upload-compacted` flag has been promoted to non hidden, non experimental state. More info available [here](docs/quick-tutorial.md#uploading-old-metrics).
+- [#1867](https://github.com/thanos-io/thanos/pull/1867) Ruler: now sets a `Thanos/$version` `User-Agent` in requests
+- [#1887](https://github.com/thanos-io/thanos/pull/1887) Service discovery now deduplicates targets between different target groups
+
+## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03
+
+### Added
+
+- [#1678](https://github.com/thanos-io/thanos/pull/1678) Add Lightstep as a tracing provider.
+- [#1687](https://github.com/thanos-io/thanos/pull/1687) Add a new `--grpc-grace-period` CLI option to components which serve gRPC to set how long to wait until gRPC Server shuts down.
+- [#1660](https://github.com/thanos-io/thanos/pull/1660) Sidecar: Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up.
+- [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss) for further information.
+- [#1680](https://github.com/thanos-io/thanos/pull/1680) Add a new `--http-grace-period` CLI option to components which serve HTTP to set how long to wait until HTTP Server shuts down.
+- [#1712](https://github.com/thanos-io/thanos/pull/1712) Bucket: Rename flag on bucket web component from `--listen` to `--http-address` to match other components.
+- [#1733](https://github.com/thanos-io/thanos/pull/1733) Compactor: New metric `thanos_compactor_iterations_total` on Thanos Compactor which shows the number of successful iterations.
+- [#1758](https://github.com/thanos-io/thanos/pull/1758) Bucket: `thanos bucket web` now supports `--web.external-prefix` for proxying on a subpath.
+- [#1770](https://github.com/thanos-io/thanos/pull/1770) Bucket: Add `--web.prefix-header` flags to allow for bucket UI to be accessible behind a reverse proxy.
+- [#1668](https://github.com/thanos-io/thanos/pull/1668) Receiver: Added TLS options for both server and client remote write.
+
+### Fixed
+
+- [#1656](https://github.com/thanos-io/thanos/pull/1656) Store Gateway: Store now starts metric and status probe HTTP server earlier in its start-up sequence. `/-/healthy` endpoint now starts to respond with success earlier. `/metrics` endpoint starts serving metrics earlier as well. Make sure to point your readiness probes to the `/-/ready` endpoint rather than `/metrics`.
+- [#1669](https://github.com/thanos-io/thanos/pull/1669) Store Gateway: Fixed store sharding. Now it does not load excluded meta.jsons and load/fetch index-cache.json files.
+- [#1670](https://github.com/thanos-io/thanos/pull/1670) Sidecar: Fixed un-ordered blocks upload. Sidecar now uploads the oldest blocks first.
+- [#1568](https://github.com/thanos-io/thanos/pull/1709) Store Gateway: Store now retains the first raw value of a chunk during downsampling to avoid losing some counter resets that occur on an aggregation boundary.
+- [#1751](https://github.com/thanos-io/thanos/pull/1751) Querier: Fixed labels for StoreUI
+- [#1773](https://github.com/thanos-io/thanos/pull/1773) Ruler: Fixed the /api/v1/rules endpoint that returned 500 status code with `failed to assert type of rule ...` message.
+- [#1770](https://github.com/thanos-io/thanos/pull/1770) Querier: Fixed `--web.external-prefix` 404s for static resources.
+- [#1785](https://github.com/thanos-io/thanos/pull/1785) Ruler: The /api/v1/rules endpoints now returns the original rule filenames.
+- [#1791](https://github.com/thanos-io/thanos/pull/1791) Ruler: Ruler now supports identical rule filenames in different directories.
+- [#1562](https://github.com/thanos-io/thanos/pull/1562) Querier: Downsampling option now carries through URL.
+- [#1675](https://github.com/thanos-io/thanos/pull/1675) Querier: Reduced resource usage while using certain queries like `offset`.
+- [#1725](https://github.com/thanos-io/thanos/pull/1725) & [#1718](https://github.com/thanos-io/thanos/pull/1718) Store Gateway: Per request memory improvements.
+
+### Changed
+
+- [#1666](https://github.com/thanos-io/thanos/pull/1666) Compact: `thanos_compact_group_compactions_total` now counts block compactions, so operations that resulted in a compacted block. The old behaviour
+is now exposed by new metric: `thanos_compact_group_compaction_runs_started_total` and `thanos_compact_group_compaction_runs_completed_total` which counts compaction runs overall.
+- [#1748](https://github.com/thanos-io/thanos/pull/1748) Updated all dependencies.
+- [#1694](https://github.com/thanos-io/thanos/pull/1694) `prober_ready` and `prober_healthy` metrics are removed, for sake of `status`. Now `status` exposes same metric with a label, `check`. `check` can have "healty" or "ready" depending on status of the probe.
+- [#1790](https://github.com/thanos-io/thanos/pull/1790) Ruler: Fixes subqueries support for ruler.
+- [#1769](https://github.com/thanos-io/thanos/pull/1769) & [#1545](https://github.com/thanos-io/thanos/pull/1545) Adjusted most of the metrics histogram buckets.
 
 ## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14
 
 ### Fixed
 
 - [#1632](https://github.com/thanos-io/thanos/issues/1632) Removes the duplicated external labels detection on Thanos Querier; warning only; Made Store Gateway compatible with older Querier versions.
- NOTE: `thanos_store_nodes_grpc_connections` metric is now per `external_labels` and `store_type`. It is a recommended  metric for Querier storeAPIs. `thanos_store_node_info` is marked as obsolete and will be removed in next release.
- NOTE2: Store Gateway is not advertising artificial: `"@thanos_compatibility_store_type=store"` label. This is to have current Store Gateway compatible with Querier pre v0.8.0. 
-This label can be disabled by hidden `debug.advertise-compatibility-label=false` flag on Store Gateway.
- 
+  * NOTE: `thanos_store_nodes_grpc_connections` metric is now per `external_labels` and `store_type`. It is a recommended  metric for Querier storeAPIs. `thanos_store_node_info` is marked as obsolete and will be removed in next release.
+  * NOTE2: Store Gateway is now advertising artificial: `"@thanos_compatibility_store_type=store"` label. This is to have the current Store Gateway compatible with Querier pre v0.8.0.
+  This label can be disabled by hidden `debug.advertise-compatibility-label=false` flag on Store Gateway.
+
 ## [v0.8.0](https://github.com/thanos-io/thanos/releases/tag/v0.8.0) - 2019.10.10
 
 Lot's of improvements this release! Noteworthy items:
 - First Katacoda tutorial! 🐱
-- Fixed Deletion order causing Compactor to produce not needed 👻 blocks with missing random files. 
+- Fixed Deletion order causing Compactor to produce not needed 👻 blocks with missing random files.
 - Store GW memory improvements (more to come!).
 - Querier allows multiple deduplication labels.
 - Both Compactor and Store Gateway can be **sharded** within the same bucket using relabelling!
@@ -38,7 +119,7 @@ both Prometheus and sidecar with Thanos: https://prometheus.io/blog/2019/10/10/r
 
 - [#1619](https://github.com/thanos-io/thanos/pull/1619) Thanos sidecar allows to limit min time range for data it exposes from Prometheus.
 - [#1583](https://github.com/thanos-io/thanos/pull/1583) Thanos sharding:
-  - Add relabel config (`--selector.relabel-config-file` and `selector.relabel-config`) into Thanos Store and Compact components. 
+  - Add relabel config (`--selector.relabel-config-file` and `selector.relabel-config`) into Thanos Store and Compact components.
 Selecting blocks to serve depends on the result of block labels relabeling.
   - For store gateway, advertise labels from "approved" blocks.
 - [#1540](https://github.com/thanos-io/thanos/pull/1540) Thanos Downsample added `/-/ready` and `/-/healthy` endpoints.
@@ -51,8 +132,8 @@ Selecting blocks to serve depends on the result of block labels relabeling.
 - [#1362](https://github.com/thanos-io/thanos/pull/1362) Optional `replicaLabels` param for `/query` and
 `/query_range` querier endpoints. When provided overwrite the `query.replica-label` cli flags.
 - [#1482](https://github.com/thanos-io/thanos/pull/1482) Thanos now supports Elastic APM as tracing provider.
-- [#1612](https://github.com/thanos-io/thanos/pull/1612) Thanos Rule added `resendDelay` flag. 
-- [#1480](https://github.com/thanos-io/thanos/pull/1480) Thanos Receive flushes storage on hashring change. 
+- [#1612](https://github.com/thanos-io/thanos/pull/1612) Thanos Rule added `resendDelay` flag.
+- [#1480](https://github.com/thanos-io/thanos/pull/1480) Thanos Receive flushes storage on hashring change.
 - [#1613](https://github.com/thanos-io/thanos/pull/1613) Thanos Receive now traces forwarded requests.
 
 ### Changed
@@ -72,7 +153,7 @@ once for multiple deduplication labels like: `--query.replica-label=prometheus_r
 - [#1544](https://github.com/thanos-io/thanos/pull/1544) Iterating over object store is resilient to the edge case for some providers.
 - [#1469](https://github.com/thanos-io/thanos/pull/1469) Fixed Azure potential failures (EOF) when requesting more data then blob has.
 - [#1512](https://github.com/thanos-io/thanos/pull/1512) Thanos Store fixed memory leak for chunk pool.
-- [#1488](https://github.com/thanos-io/thanos/pull/1488) Thanos Rule now now correctly links to query URL from rules and alerts. 
+- [#1488](https://github.com/thanos-io/thanos/pull/1488) Thanos Rule now now correctly links to query URL from rules and alerts.
 
 ## [v0.7.0](https://github.com/thanos-io/thanos/releases/tag/v0.7.0) - 2019.09.02
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -51,12 +51,8 @@ $ make build
 $ <Iterate your development>
 $ git push origin <your_branch_for_new_pr>
 ```
-5. If you don't have a live object store ready add these envvars to skip tests for these:
-- THANOS_SKIP_GCS_TESTS to skip GCS tests.
-- THANOS_SKIP_S3_AWS_TESTS to skip AWS tests.
-- THANOS_SKIP_AZURE_TESTS to skip Azure tests.
-- THANOS_SKIP_SWIFT_TESTS to skip SWIFT tests.
-- THANOS_SKIP_TENCENT_COS_TESTS to skip Tencent COS tests.
+5. If you don't have a live object store ready add this envvar to skip tests for these:
+- THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS
 
 If you skip all of these, the store specific tests will be run against memory object storage only.
 CI runs GCS and inmem tests only for now. Not having these variables will produce auth errors against GCS, AWS, Azure or COS tests.