Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added binary index header implementation. #1952

Merged
merged 4 commits into from
Jan 21, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions cmd/thanos/store.go
Original file line number Diff line number Diff line change
@@ -75,6 +75,9 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) {

selectorRelabelConf := regSelectorRelabelFlags(cmd)

enableIndexHeader := cmd.Flag("experimental.enable-index-header", "If true, Store Gateway will recreate index-header instead of index-cache.json for each block. This will replace index-cache.json permanently once it will be out of experimental stage.").
Hidden().Default("false").Bool()

m[component.Store.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, debugLogging bool) error {
if minTime.PrometheusTimestamp() > maxTime.PrometheusTimestamp() {
return errors.Errorf("invalid argument: --min-time '%s' can't be greater than --max-time '%s'",
@@ -109,6 +112,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) {
},
selectorRelabelConf,
*advertiseCompatibilityLabel,
*enableIndexHeader,
)
}
}
@@ -140,6 +144,7 @@ func runStore(
filterConf *store.FilterConfig,
selectorRelabelConf *extflag.PathOrContent,
advertiseCompatibilityLabel bool,
enableIndexHeader bool,
) error {
// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
statusProber := prober.New(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
@@ -214,6 +219,9 @@ func runStore(
return errors.Wrap(err, "meta fetcher")
}

if enableIndexHeader {
level.Info(logger).Log("msg", "index-header instead of index-cache.json enabled")
}
bs, err := store.NewBucketStore(
logger,
reg,
@@ -228,6 +236,7 @@ func runStore(
blockSyncConcurrency,
filterConf,
advertiseCompatibilityLabel,
enableIndexHeader,
)
if err != nil {
return errors.Wrap(err, "create object storage store")
59 changes: 59 additions & 0 deletions docs/components/store.md
Original file line number Diff line number Diff line change
@@ -221,3 +221,62 @@ While the remaining settings are **optional**:
- `max_get_multi_concurrency`: maximum number of concurrent connections when fetching keys. If set to `0`, the concurrency is unlimited.
- `max_get_multi_batch_size`: maximum number of keys a single underlying operation should fetch. If more keys are specified, internally keys are splitted into multiple batches and fetched concurrently, honoring `max_get_multi_concurrency`. If set to `0`, the batch size is unlimited.
- `dns_provider_update_interval`: the DNS discovery update interval.


## Index Header

In order to query series inside blocks from object storage, Store Gateway has to know certain initial info about each block such as:

* symbols table to unintern string values
* postings offset for posting lookup

In order to achieve so, on startup for each block `index-header` is built from pieces of original block's index and stored on disk.
Such `index-header` file is then mmaped and used by Store Gateway.

### Format (version 1)

The following describes the format of the `index-header` file found in each block store gateway local directory.
It is terminated by a table of contents which serves as an entry point into the index.

```
┌─────────────────────────────┬───────────────────────────────┐
│ magic(0xBAAAD792) <4b> │ version(1) <1 byte> │
├─────────────────────────────┬───────────────────────────────┤
│ index version(2) <1 byte> │ index PostingOffsetTable <8b> │
├─────────────────────────────┴───────────────────────────────┤
│ ┌─────────────────────────────────────────────────────────┐ │
│ │ Symbol Table (exact copy from original index) │ │
│ ├─────────────────────────────────────────────────────────┤ │
│ │ Posting Offset Table (exact copy from index) │ │
│ ├─────────────────────────────────────────────────────────┤ │
│ │ TOC │ │
│ └─────────────────────────────────────────────────────────┘ │
└─────────────────────────────────────────────────────────────┘
```
When the index is written, an arbitrary number of padding bytes may be added between the lined out main sections above. When sequentially scanning through the file, any zero bytes after a section's specified length must be skipped.
Most of the sections described below start with a `len` field. It always specifies the number of bytes just before the trailing CRC32 checksum. The checksum is always calculated over those `len` bytes.
### Symbol Table
See [Symbols](https://github.com/prometheus/prometheus/blob/d782387f814753b0118d402ec8cdbdef01bf9079/tsdb/docs/format/index.md#symbol-table)
### Postings Offset Table
See [Posting Offset Table](https://github.com/prometheus/prometheus/blob/d782387f814753b0118d402ec8cdbdef01bf9079/tsdb/docs/format/index.md#postings-offset-table)
### TOC
The table of contents serves as an entry point to the entire index and points to various sections in the file.
If a reference is zero, it indicates the respective section does not exist and empty results should be returned upon lookup.
```
┌─────────────────────────────────────────┐
│ ref(symbols) <8b> │
├─────────────────────────────────────────┤
│ ref(postings offset table) <8b> │
├─────────────────────────────────────────┤
│ CRC32 <4b> │
└─────────────────────────────────────────┘
```
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -72,7 +72,7 @@ require (
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
github.com/prometheus/common v0.7.0
github.com/prometheus/procfs v0.0.6 // indirect
github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef // master ~ v2.15.2
github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 // master ~ v2.15.2
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect
github.com/satori/go.uuid v1.2.0 // indirect
github.com/smartystreets/assertions v1.0.1 // indirect
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
@@ -501,8 +501,8 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
github.com/prometheus/procfs v0.0.6 h1:0qbH+Yqu/cj1ViVLvEWCP6qMQ4efWUj6bQqOEA0V0U4=
github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s=
github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef h1:pYYKXo/zGx25kyViw+Gdbxd0ItIg+vkVKpwgWUEyIc4=
github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI=
github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 h1:uEq+8hKI4kfycPLSKNw844YYkdMNpC2eZpov73AvlFk=
github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
4 changes: 3 additions & 1 deletion pkg/block/block.go
Original file line number Diff line number Diff line change
@@ -27,8 +27,10 @@ const (
MetaFilename = "meta.json"
// IndexFilename is the known index file for block index.
IndexFilename = "index"
// IndexCacheFilename is the canonical name for index cache file that stores essential information needed.
// IndexCacheFilename is the canonical name for json index cache file that stores essential information.
IndexCacheFilename = "index.cache.json"
// IndexHeaderFilename is the canonical name for binary index header file that stores essential information.
IndexHeaderFilename = "index-header"
// ChunksDirname is the known dir name for chunks with compressed samples.
ChunksDirname = "chunks"

35 changes: 4 additions & 31 deletions pkg/block/block_test.go
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@ package block

import (
"context"
"io"
"io/ioutil"
"os"
"path"
@@ -12,7 +11,6 @@ import (

"github.com/fortytw2/leaktest"
"github.com/go-kit/kit/log"
"github.com/pkg/errors"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/thanos-io/thanos/pkg/objstore/inmem"
"github.com/thanos-io/thanos/pkg/testutil"
@@ -104,7 +102,7 @@ func TestUpload(t *testing.T) {
testutil.NotOk(t, err)
testutil.Assert(t, strings.HasSuffix(err.Error(), "/meta.json: no such file or directory"), "")
}
testutil.Ok(t, cpy(path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename)))
testutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename))
{
// Missing chunks.
err := Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()))
@@ -115,7 +113,7 @@ func TestUpload(t *testing.T) {
testutil.Equals(t, 1, len(bkt.Objects()))
}
testutil.Ok(t, os.MkdirAll(path.Join(tmpDir, "test", b1.String(), ChunksDirname), os.ModePerm))
testutil.Ok(t, cpy(path.Join(tmpDir, b1.String(), ChunksDirname, "000001"), path.Join(tmpDir, "test", b1.String(), ChunksDirname, "000001")))
testutil.Copy(t, path.Join(tmpDir, b1.String(), ChunksDirname, "000001"), path.Join(tmpDir, "test", b1.String(), ChunksDirname, "000001"))
{
// Missing index file.
err := Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()))
@@ -125,7 +123,7 @@ func TestUpload(t *testing.T) {
// Only debug meta.json present.
testutil.Equals(t, 1, len(bkt.Objects()))
}
testutil.Ok(t, cpy(path.Join(tmpDir, b1.String(), IndexFilename), path.Join(tmpDir, "test", b1.String(), IndexFilename)))
testutil.Copy(t, path.Join(tmpDir, b1.String(), IndexFilename), path.Join(tmpDir, "test", b1.String(), IndexFilename))
testutil.Ok(t, os.Remove(path.Join(tmpDir, "test", b1.String(), MetaFilename)))
{
// Missing meta.json file.
@@ -136,7 +134,7 @@ func TestUpload(t *testing.T) {
// Only debug meta.json present.
testutil.Equals(t, 1, len(bkt.Objects()))
}
testutil.Ok(t, cpy(path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename)))
testutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename))
{
// Full block.
testutil.Ok(t, Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String())))
@@ -170,31 +168,6 @@ func TestUpload(t *testing.T) {
}
}

func cpy(src, dst string) error {
sourceFileStat, err := os.Stat(src)
if err != nil {
return err
}

if !sourceFileStat.Mode().IsRegular() {
return errors.Errorf("%s is not a regular file", src)
}

source, err := os.Open(src)
if err != nil {
return err
}
defer source.Close()

destination, err := os.Create(dst)
if err != nil {
return err
}
defer destination.Close()
_, err = io.Copy(destination, source)
return err
}

func TestDelete(t *testing.T) {
defer leaktest.CheckTimeout(t, 10*time.Second)()

Loading