Skip to content

Commit

Permalink
Docker driver relabeling (#2199)
Browse files Browse the repository at this point in the history 
* Add relabeling to docker driver using `loki-relabel-config` log option.

Signed-off-by: Cyril Tovena <[email protected]>

* Add docs.

Signed-off-by: Cyril Tovena <[email protected]>
  • Loading branch information
@cyriltovena
cyriltovena authored Jun 19, 2020
1 parent 1729592 commit 79a90dc
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 3 deletions.
29 changes: 28 additions & 1 deletion cmd/docker-driver/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,31 @@ This is a bit more difficult as you need to properly escape bash special charact

Providing both `loki-pipeline-stage-file` and `loki-pipeline-stages` will cause an error.

## Relabeling

You can use [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) configuration to modify labels discovered by the driver. The configuration must be passed as a YAML string like the [pipeline stages](#pipeline-stages).

For example the configuration below will rename the label `swarm_stack` and `swarm_service` to respectively `namespace` and `service`.

```yaml
version: "3"
services:
nginx:
image: grafana/grafana
logging:
driver: loki
options:
loki-url: http://host.docker.internal:3100/loki/api/v1/push
loki-relabel-config: |
- action: labelmap
regex: swarm_stack
replacement: namespace
- action: labelmap
regex: swarm_(service)
ports:
- "3000:3000"
```

## log-opt options

To specify additional logging driver options, you can use the --log-opt NAME=VALUE flag.
Expand All @@ -174,7 +199,8 @@ To specify additional logging driver options, you can use the --log-opt NAME=VAL
| `loki-max-backoff` | No | `10s` | The maximum amount of time to wait before retrying a batch. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". |
| `loki-retries` | No | `10` | The maximum amount of retries for a log batch. |
| `loki-pipeline-stage-file` | No | | The location of a pipeline stage configuration file ([example](./pipeline-example.yaml)). Pipeline stages allows to parse log lines to extract more labels. [see documentation](../../docs/logentry/processing-log-lines.md) |
| `loki-pipeline-stages` | No | | The pipeline stage configuration provided as a string [see](#pipeline-stages) and [see documentation](../../docs/clients/promtail/pipelines.md) |
| `loki-pipeline-stages` | No | | The pipeline stage configuration provided as a string [see](#pipeline-stages) and [see documentation](../../docs/clients/promtail/pipelines.md) |
| `loki-relabel-config` | No | | A [Prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) allowing you to rename labels [see](#relabeling) |
| `loki-tenant-id` | No | | Set the tenant id (http header`X-Scope-OrgID`) when sending logs to Loki. It can be overrides by a pipeline stage. |
| `loki-tls-ca-file` | No | | Set the path to a custom certificate authority. |
| `loki-tls-cert-file` | No | | Set the path to a client certificate file. |
Expand All @@ -190,6 +216,7 @@ To specify additional logging driver options, you can use the --log-opt NAME=VAL
| `env` | No | | Comma-separated list of keys of environment variables to be included in message if they specified for a container. |
| `env-regex` | No | | A regular expression to match logging-related environment variables. Used for advanced log label options. If there is collision between the label and env keys, the value of the env takes precedence. Both options add additional fields to the labels of a logging message. |


## Uninstall the plugin

To cleanly disable and remove the plugin, run:
Expand Down
27 changes: 25 additions & 2 deletions cmd/docker-driver/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,20 @@ import (
"strings"
"time"

"github.com/cortexproject/cortex/pkg/util"
cortex_util "github.com/cortexproject/cortex/pkg/util"
"github.com/cortexproject/cortex/pkg/util/flagext"
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/daemon/logger/templates"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/relabel"
yaml "gopkg.in/yaml.v2"

"github.com/grafana/loki/pkg/helpers"
"github.com/grafana/loki/pkg/logentry/stages"
"github.com/grafana/loki/pkg/promtail/client"
"github.com/grafana/loki/pkg/promtail/targets"
"github.com/grafana/loki/pkg/util"
)

const (
Expand All @@ -44,6 +47,7 @@ const (
cfgTenantIDKey = "loki-tenant-id"
cfgNofile = "no-file"
cfgKeepFile = "keep-file"
cfgRelabelKey = "loki-relabel-config"

swarmServiceLabelKey = "com.docker.swarm.service.name"
swarmStackLabelKey = "com.docker.stack.namespace"
Expand All @@ -65,7 +69,7 @@ var (
defaultClientConfig = client.Config{
BatchWait: 1 * time.Second,
BatchSize: 100 * 1024,
BackoffConfig: util.BackoffConfig{
BackoffConfig: cortex_util.BackoffConfig{
MinBackoff: 100 * time.Millisecond,
MaxBackoff: 10 * time.Second,
MaxRetries: 10,
Expand Down Expand Up @@ -106,6 +110,7 @@ func validateDriverOpt(loggerInfo logger.Info) error {
case cfgPipelineStagesKey:
case cfgPipelineStagesFileKey:
case cfgTenantIDKey:
case cfgRelabelKey:
case cfgNofile:
case cfgKeepFile:
case "labels":
Expand Down Expand Up @@ -280,6 +285,15 @@ func parseConfig(logCtx logger.Info) (*config, error) {
}
labels[targets.FilenameLabel] = model.LabelValue(logCtx.LogPath)

// Process relabel configs.
if relabelString, ok := logCtx.Config[cfgRelabelKey]; ok && relabelString != "" {
relabeled, err := relabelConfig(relabelString, labels)
if err != nil {
return nil, fmt.Errorf("error applying relabel config: %s err:%s", relabelString, err)
}
labels = relabeled
}

// parse pipeline stages
pipeline, err := parsePipeline(logCtx)
if err != nil {
Expand Down Expand Up @@ -347,6 +361,15 @@ func parseInt(key string, logCtx logger.Info, set func(i int)) error {
return nil
}

func relabelConfig(config string, lbs model.LabelSet) (model.LabelSet, error) {
relabelConfig := make([]*relabel.Config, 0)
if err := yaml.UnmarshalStrict([]byte(config), &relabelConfig); err != nil {
return nil, err
}
relabed := relabel.Process(labels.FromMap(util.ModelLabelSetToMap(lbs)), relabelConfig...)
return model.LabelSet(util.LabelsToMetric(relabed)), nil
}

func parseBoolean(key string, logCtx logger.Info, defaultValue bool) (bool, error) {
value, ok := logCtx.Config[key]
if !ok || value == "" {
Expand Down
45 changes: 45 additions & 0 deletions cmd/docker-driver/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,55 @@ import (
"github.com/cortexproject/cortex/pkg/util"
"github.com/docker/docker/daemon/logger"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"

"github.com/grafana/loki/pkg/logentry/stages"
)

var jobRename = `
- regex: (.*)
source_labels: [swarm_stack]
target_label: job
- regex: ^swarm_stack$
action: labeldrop`

func Test_relabelConfig(t *testing.T) {

tests := []struct {
name string
config string
in model.LabelSet
out model.LabelSet
wantErr bool
}{
{
"config err",
"foo",
nil,
nil,
true,
},
{
"config err",
jobRename,
model.LabelSet{"swarm_stack": "foo", "bar": "buzz"},
model.LabelSet{"job": "foo", "bar": "buzz"},
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := relabelConfig(tt.config, tt.in)
if (err != nil) != tt.wantErr {
t.Errorf("relabelConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
require.True(t, got.Equal(tt.out))
})
}
}

var pipelineString = `
- regex:
expression: '(level|lvl|severity)=(?P<level>\w+)'
Expand Down

0 comments on commit 79a90dc

Please sign in to comment.