From 328939cbbaed1975b8d52540fa17df34a8e4e2f7 Mon Sep 17 00:00:00 2001 From: WalkerWang731 Date: Tue, 13 Apr 2021 18:13:19 +0800 Subject: [PATCH 1/8] add index roration for ES storage Signed-off-by: WalkerWang731 --- pkg/es/config/config.go | 1 + plugin/storage/es/options.go | 29 ++++++++++++++++++++++++--- plugin/storage/es/options_test.go | 24 ++++++++++++++++++++++ plugin/storage/es/spanstore/reader.go | 8 ++++++-- 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index 38f3fc42f3f..6d8ec95d00a 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -63,6 +63,7 @@ type Configuration struct { BulkFlushInterval time.Duration `mapstructure:"-"` IndexPrefix string `mapstructure:"index_prefix"` IndexDateLayout string `mapstructure:"index_date_layout"` + IndexRotate string `mapstructure:"-"` Tags TagsAsFields `mapstructure:"tags_as_fields"` Enabled bool `mapstructure:"-"` TLS tlscfg.Options `mapstructure:"tls"` diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index da2702f3cd7..624d5ce6f1b 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -46,6 +46,7 @@ const ( suffixTimeout = ".timeout" suffixIndexPrefix = ".index-prefix" suffixIndexDateSeparator = ".index-date-separator" + suffixIndexRotate = ".index-rotate" suffixTagsAsFields = ".tags-as-fields" suffixTagsAsFieldsAll = suffixTagsAsFields + ".all" suffixTagsAsFieldsInclude = suffixTagsAsFields + ".include" @@ -65,6 +66,8 @@ const ( defaultRemoteReadClusters = "" // default separator for Elasticsearch index date layout. defaultIndexDateSeparator = "-" + + defaultIndexRotate = "day" ) // TODO this should be moved next to config.Configuration struct (maybe ./flags package) @@ -205,7 +208,11 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { flagSet.String( nsConfig.namespace+suffixIndexDateSeparator, defaultIndexDateSeparator, - "Optional date separator of Jaeger indices. For example \".\" creates \"jaeger-span-2020.11.20 \".") + "Optional date separator of Jaeger indices. For example \".\" creates \"jaeger-span-2020.11.20\".") + flagSet.String( + nsConfig.namespace+suffixIndexRotate, + defaultIndexRotate, + "Optional rotation opportunity of Jaeger indices. For example \"day\" creates \"jaeger-span-yyyy-HH-dd\" every day after UTC 12AM. Valid opportunity: [hour, day]") flagSet.Bool( nsConfig.namespace+suffixTagsAsFieldsAll, nsConfig.Tags.AllAsFields, @@ -295,7 +302,6 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { cfg.BulkFlushInterval = v.GetDuration(cfg.namespace + suffixBulkFlushInterval) cfg.Timeout = v.GetDuration(cfg.namespace + suffixTimeout) cfg.IndexPrefix = v.GetString(cfg.namespace + suffixIndexPrefix) - cfg.IndexDateLayout = initDateLayout(v.GetString(cfg.namespace + suffixIndexDateSeparator)) cfg.Tags.AllAsFields = v.GetBool(cfg.namespace + suffixTagsAsFieldsAll) cfg.Tags.Include = v.GetString(cfg.namespace + suffixTagsAsFieldsInclude) cfg.Tags.File = v.GetString(cfg.namespace + suffixTagsFile) @@ -317,6 +323,19 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { if len(remoteReadClusters) > 0 { cfg.RemoteReadClusters = strings.Split(remoteReadClusters, ",") } + + indexRotate := strings.ToLower(v.GetString(cfg.namespace + suffixIndexRotate)) + + switch indexRotate { + case "day": + cfg.IndexDateLayout = initDateLayoutDay(v.GetString(cfg.namespace + suffixIndexDateSeparator)) + + case "hour": + cfg.IndexDateLayout = initDateLayoutHours(v.GetString(cfg.namespace + suffixIndexDateSeparator)) + + default: + cfg.IndexDateLayout = initDateLayoutDay(v.GetString(cfg.namespace + suffixIndexDateSeparator)) + } } // GetPrimary returns primary configuration. @@ -343,6 +362,10 @@ func stripWhiteSpace(str string) string { return strings.Replace(str, " ", "", -1) } -func initDateLayout(separator string) string { +func initDateLayoutDay(separator string) string { return fmt.Sprintf("2006%s01%s02", separator, separator) } + +func initDateLayoutHours(separator string) string { + return fmt.Sprintf("2006%s01%s02%s15", separator, separator, separator) +} diff --git a/plugin/storage/es/options_test.go b/plugin/storage/es/options_test.go index d9bee8bd2b6..a223b760e3b 100644 --- a/plugin/storage/es/options_test.go +++ b/plugin/storage/es/options_test.go @@ -174,3 +174,27 @@ func TestIndexDateSeparator(t *testing.T) { }) } } + +func TestIndexRotate(t *testing.T) { + testCases := []struct { + name string + flags []string + wantDateLayout string + }{ + {"not defined (default)", []string{}, "2006-01-02"}, + {"day rotation", []string{"--es.index-rotate=day"}, "2006-01-02"}, + {"hour rotation", []string{"--es.index-rotate=hour"}, "2006-01-02-15"}, + {"error rotation change default", []string{"--es.index-rotate=hours"}, "2006-01-02"}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts := NewOptions("es") + v, command := config.Viperize(opts.AddFlags) + command.ParseFlags(tc.flags) + opts.InitFromViper(v) + primary := opts.GetPrimary() + assert.Equal(t, tc.wantDateLayout, primary.IndexDateLayout) + + }) + } +} diff --git a/plugin/storage/es/spanstore/reader.go b/plugin/storage/es/spanstore/reader.go index e1bee4e6647..d946c2f4c44 100644 --- a/plugin/storage/es/spanstore/reader.go +++ b/plugin/storage/es/spanstore/reader.go @@ -200,8 +200,12 @@ func timeRangeIndices(indexName, indexDateLayout string, startTime time.Time, en firstIndex := indexWithDate(indexName, indexDateLayout, startTime) currentIndex := indexWithDate(indexName, indexDateLayout, endTime) for currentIndex != firstIndex { - indices = append(indices, currentIndex) - endTime = endTime.Add(-24 * time.Hour) + if len(indices) == 0 { + indices = append(indices, currentIndex) + } else if indices[len(indices)-1] != currentIndex { + indices = append(indices, currentIndex) + } + endTime = endTime.Add(-1 * time.Hour) currentIndex = indexWithDate(indexName, indexDateLayout, endTime) } indices = append(indices, firstIndex) From 1e2affd397b259b7794973928e239b710addd4ca Mon Sep 17 00:00:00 2001 From: WalkerWang731 Date: Tue, 20 Apr 2021 20:19:47 +0800 Subject: [PATCH 2/8] adjust terminology from rotate to rollover and simplified code Signed-off-by: WalkerWang731 --- pkg/es/config/config.go | 2 +- plugin/storage/es/options.go | 39 +++++++++++---------------- plugin/storage/es/options_test.go | 8 +++--- plugin/storage/es/spanstore/reader.go | 4 +-- 4 files changed, 21 insertions(+), 32 deletions(-) diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index 6d8ec95d00a..c096c0b9bbe 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -63,7 +63,7 @@ type Configuration struct { BulkFlushInterval time.Duration `mapstructure:"-"` IndexPrefix string `mapstructure:"index_prefix"` IndexDateLayout string `mapstructure:"index_date_layout"` - IndexRotate string `mapstructure:"-"` + IndexRollover string `mapstructure:"-"` Tags TagsAsFields `mapstructure:"tags_as_fields"` Enabled bool `mapstructure:"-"` TLS tlscfg.Options `mapstructure:"tls"` diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 624d5ce6f1b..66f5a9f3918 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -46,7 +46,7 @@ const ( suffixTimeout = ".timeout" suffixIndexPrefix = ".index-prefix" suffixIndexDateSeparator = ".index-date-separator" - suffixIndexRotate = ".index-rotate" + suffixIndexRollover = ".index-rollover" suffixTagsAsFields = ".tags-as-fields" suffixTagsAsFieldsAll = suffixTagsAsFields + ".all" suffixTagsAsFieldsInclude = suffixTagsAsFields + ".include" @@ -67,7 +67,7 @@ const ( // default separator for Elasticsearch index date layout. defaultIndexDateSeparator = "-" - defaultIndexRotate = "day" + defaultIndexRollover = "day" ) // TODO this should be moved next to config.Configuration struct (maybe ./flags package) @@ -210,9 +210,10 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { defaultIndexDateSeparator, "Optional date separator of Jaeger indices. For example \".\" creates \"jaeger-span-2020.11.20\".") flagSet.String( - nsConfig.namespace+suffixIndexRotate, - defaultIndexRotate, - "Optional rotation opportunity of Jaeger indices. For example \"day\" creates \"jaeger-span-yyyy-HH-dd\" every day after UTC 12AM. Valid opportunity: [hour, day]") + nsConfig.namespace+suffixIndexRollover, + defaultIndexRollover, + "Rotates Jaeger indices over the given period. For example \"day\" creates \"jaeger-span-yyyy-HH-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ + "Jaeger also support Elasticsearch ILM to manage indices, reference(https://www.jaegertracing.io/docs/deployment/#elasticsearch-ilm-support)") flagSet.Bool( nsConfig.namespace+suffixTagsAsFieldsAll, nsConfig.Tags.AllAsFields, @@ -302,6 +303,7 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { cfg.BulkFlushInterval = v.GetDuration(cfg.namespace + suffixBulkFlushInterval) cfg.Timeout = v.GetDuration(cfg.namespace + suffixTimeout) cfg.IndexPrefix = v.GetString(cfg.namespace + suffixIndexPrefix) + cfg.IndexDateLayout = initDateLayout(strings.ToLower(v.GetString(cfg.namespace+suffixIndexRollover)), v.GetString(cfg.namespace+suffixIndexDateSeparator)) cfg.Tags.AllAsFields = v.GetBool(cfg.namespace + suffixTagsAsFieldsAll) cfg.Tags.Include = v.GetString(cfg.namespace + suffixTagsAsFieldsInclude) cfg.Tags.File = v.GetString(cfg.namespace + suffixTagsFile) @@ -323,19 +325,6 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { if len(remoteReadClusters) > 0 { cfg.RemoteReadClusters = strings.Split(remoteReadClusters, ",") } - - indexRotate := strings.ToLower(v.GetString(cfg.namespace + suffixIndexRotate)) - - switch indexRotate { - case "day": - cfg.IndexDateLayout = initDateLayoutDay(v.GetString(cfg.namespace + suffixIndexDateSeparator)) - - case "hour": - cfg.IndexDateLayout = initDateLayoutHours(v.GetString(cfg.namespace + suffixIndexDateSeparator)) - - default: - cfg.IndexDateLayout = initDateLayoutDay(v.GetString(cfg.namespace + suffixIndexDateSeparator)) - } } // GetPrimary returns primary configuration. @@ -362,10 +351,12 @@ func stripWhiteSpace(str string) string { return strings.Replace(str, " ", "", -1) } -func initDateLayoutDay(separator string) string { - return fmt.Sprintf("2006%s01%s02", separator, separator) -} - -func initDateLayoutHours(separator string) string { - return fmt.Sprintf("2006%s01%s02%s15", separator, separator, separator) +func initDateLayout(RolloverBy, separator string) string { + seps := []interface{}{separator, separator} + indexLayout := "2006%s01%s02" + if RolloverBy == "hour" { + indexLayout = "2006%s01%s02%s15" + seps = append(seps, separator) + } + return fmt.Sprintf(indexLayout, seps...) } diff --git a/plugin/storage/es/options_test.go b/plugin/storage/es/options_test.go index a223b760e3b..b98b7c9b865 100644 --- a/plugin/storage/es/options_test.go +++ b/plugin/storage/es/options_test.go @@ -175,16 +175,16 @@ func TestIndexDateSeparator(t *testing.T) { } } -func TestIndexRotate(t *testing.T) { +func TestIndexRollover(t *testing.T) { testCases := []struct { name string flags []string wantDateLayout string }{ {"not defined (default)", []string{}, "2006-01-02"}, - {"day rotation", []string{"--es.index-rotate=day"}, "2006-01-02"}, - {"hour rotation", []string{"--es.index-rotate=hour"}, "2006-01-02-15"}, - {"error rotation change default", []string{"--es.index-rotate=hours"}, "2006-01-02"}, + {"day rotation", []string{"--es.index-rollover=day"}, "2006-01-02"}, + {"hour rotation", []string{"--es.index-rollover=hour"}, "2006-01-02-15"}, + {"error rotation change default", []string{"--es.index-rollover=hours"}, "2006-01-02"}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/plugin/storage/es/spanstore/reader.go b/plugin/storage/es/spanstore/reader.go index d946c2f4c44..9c59a5791bf 100644 --- a/plugin/storage/es/spanstore/reader.go +++ b/plugin/storage/es/spanstore/reader.go @@ -200,9 +200,7 @@ func timeRangeIndices(indexName, indexDateLayout string, startTime time.Time, en firstIndex := indexWithDate(indexName, indexDateLayout, startTime) currentIndex := indexWithDate(indexName, indexDateLayout, endTime) for currentIndex != firstIndex { - if len(indices) == 0 { - indices = append(indices, currentIndex) - } else if indices[len(indices)-1] != currentIndex { + if len(indices) == 0 || indices[len(indices)-1] != currentIndex { indices = append(indices, currentIndex) } endTime = endTime.Add(-1 * time.Hour) From ff61a0291ce6cafbbe6c9eca6aae154be32dd802 Mon Sep 17 00:00:00 2001 From: WalkerWang731 Date: Thu, 22 Apr 2021 14:40:14 +0800 Subject: [PATCH 3/8] update help info and make the code clearer Signed-off-by: WalkerWang731 --- plugin/storage/es/options.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 66f5a9f3918..4eb73f36478 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -212,8 +212,8 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { flagSet.String( nsConfig.namespace+suffixIndexRollover, defaultIndexRollover, - "Rotates Jaeger indices over the given period. For example \"day\" creates \"jaeger-span-yyyy-HH-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ - "Jaeger also support Elasticsearch ILM to manage indices, reference(https://www.jaegertracing.io/docs/deployment/#elasticsearch-ilm-support)") + "Rotates Jaeger indices over the given period. For example \"day\" creates \"jaeger-span-yyyy-MM-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ + "Jaeger additionally supports manual and automated (via ILM) index management. Reference: https://www.jaegertracing.io/docs/deployment/#elasticsearch-rollover.") flagSet.Bool( nsConfig.namespace+suffixTagsAsFieldsAll, nsConfig.Tags.AllAsFields, @@ -303,7 +303,6 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { cfg.BulkFlushInterval = v.GetDuration(cfg.namespace + suffixBulkFlushInterval) cfg.Timeout = v.GetDuration(cfg.namespace + suffixTimeout) cfg.IndexPrefix = v.GetString(cfg.namespace + suffixIndexPrefix) - cfg.IndexDateLayout = initDateLayout(strings.ToLower(v.GetString(cfg.namespace+suffixIndexRollover)), v.GetString(cfg.namespace+suffixIndexDateSeparator)) cfg.Tags.AllAsFields = v.GetBool(cfg.namespace + suffixTagsAsFieldsAll) cfg.Tags.Include = v.GetString(cfg.namespace + suffixTagsAsFieldsInclude) cfg.Tags.File = v.GetString(cfg.namespace + suffixTagsFile) @@ -325,6 +324,10 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { if len(remoteReadClusters) > 0 { cfg.RemoteReadClusters = strings.Split(remoteReadClusters, ",") } + + rolloverBy := strings.ToLower(v.GetString(cfg.namespace + suffixIndexRollover)) + separator := v.GetString(cfg.namespace + suffixIndexDateSeparator) + cfg.IndexDateLayout = initDateLayout(rolloverBy, separator) } // GetPrimary returns primary configuration. @@ -351,10 +354,13 @@ func stripWhiteSpace(str string) string { return strings.Replace(str, " ", "", -1) } -func initDateLayout(RolloverBy, separator string) string { +func initDateLayout(rolloverBy, separator string) string { seps := []interface{}{separator, separator} + + // Default "day" index layout. indexLayout := "2006%s01%s02" - if RolloverBy == "hour" { + + if rolloverBy == "hour" { indexLayout = "2006%s01%s02%s15" seps = append(seps, separator) } From 29ca0e164ecc0bb7cfd44c650a5e9485174fb7df Mon Sep 17 00:00:00 2001 From: WalkerWang731 Date: Mon, 26 Apr 2021 14:56:44 +0800 Subject: [PATCH 4/8] update the suggestion from yurishkuro Co-authored-by: Yuri Shkuro --- plugin/storage/es/options.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 4eb73f36478..48b8798af44 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -354,15 +354,12 @@ func stripWhiteSpace(str string) string { return strings.Replace(str, " ", "", -1) } -func initDateLayout(rolloverBy, separator string) string { - seps := []interface{}{separator, separator} - - // Default "day" index layout. - indexLayout := "2006%s01%s02" - - if rolloverBy == "hour" { - indexLayout = "2006%s01%s02%s15" - seps = append(seps, separator) - } - return fmt.Sprintf(indexLayout, seps...) +func initDateLayout(rolloverFreq, sep string) string { + // default to daily format + indexLayout := "2006" + sep + "01" + sep + "02" + if rolloverFreq == "hour" { + indexLayout = indexLayout + sep + "15" + } + return indexLayout +} } From 0435ed9be0c34434733b08088032e2bd8cc7179b Mon Sep 17 00:00:00 2001 From: WalkerWang731 Date: Mon, 26 Apr 2021 15:44:23 +0800 Subject: [PATCH 5/8] rename index rollover to index frequency and make timeRangeIndices more efficient Signed-off-by: WalkerWang731 --- pkg/es/config/config.go | 56 +++++++++--------- plugin/storage/es/options.go | 84 +++++++++++++-------------- plugin/storage/es/options_test.go | 4 +- plugin/storage/es/spanstore/reader.go | 8 ++- 4 files changed, 78 insertions(+), 74 deletions(-) diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index c096c0b9bbe..548853a8ca5 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -44,34 +44,34 @@ import ( // Configuration describes the configuration properties needed to connect to an ElasticSearch cluster type Configuration struct { - Servers []string `mapstructure:"server_urls"` - RemoteReadClusters []string `mapstructure:"remote_read_clusters"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password" json:"-"` - TokenFilePath string `mapstructure:"token_file"` - AllowTokenFromContext bool `mapstructure:"-"` - Sniffer bool `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing - SnifferTLSEnabled bool `mapstructure:"sniffer_tls_enabled"` - MaxDocCount int `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query - MaxSpanAge time.Duration `yaml:"max_span_age" mapstructure:"-"` // configures the maximum lookback on span reads - NumShards int64 `yaml:"shards" mapstructure:"num_shards"` - NumReplicas int64 `yaml:"replicas" mapstructure:"num_replicas"` - Timeout time.Duration `validate:"min=500" mapstructure:"-"` - BulkSize int `mapstructure:"-"` - BulkWorkers int `mapstructure:"-"` - BulkActions int `mapstructure:"-"` - BulkFlushInterval time.Duration `mapstructure:"-"` - IndexPrefix string `mapstructure:"index_prefix"` - IndexDateLayout string `mapstructure:"index_date_layout"` - IndexRollover string `mapstructure:"-"` - Tags TagsAsFields `mapstructure:"tags_as_fields"` - Enabled bool `mapstructure:"-"` - TLS tlscfg.Options `mapstructure:"tls"` - UseReadWriteAliases bool `mapstructure:"use_aliases"` - CreateIndexTemplates bool `mapstructure:"create_mappings"` - UseILM bool `mapstructure:"use_ilm"` - Version uint `mapstructure:"version"` - LogLevel string `mapstructure:"log_level"` + Servers []string `mapstructure:"server_urls"` + RemoteReadClusters []string `mapstructure:"remote_read_clusters"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password" json:"-"` + TokenFilePath string `mapstructure:"token_file"` + AllowTokenFromContext bool `mapstructure:"-"` + Sniffer bool `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing + SnifferTLSEnabled bool `mapstructure:"sniffer_tls_enabled"` + MaxDocCount int `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query + MaxSpanAge time.Duration `yaml:"max_span_age" mapstructure:"-"` // configures the maximum lookback on span reads + NumShards int64 `yaml:"shards" mapstructure:"num_shards"` + NumReplicas int64 `yaml:"replicas" mapstructure:"num_replicas"` + Timeout time.Duration `validate:"min=500" mapstructure:"-"` + BulkSize int `mapstructure:"-"` + BulkWorkers int `mapstructure:"-"` + BulkActions int `mapstructure:"-"` + BulkFlushInterval time.Duration `mapstructure:"-"` + IndexPrefix string `mapstructure:"index_prefix"` + IndexDateLayout string `mapstructure:"index_date_layout"` + IndexRolloverFrequency string `mapstructure:"-"` + Tags TagsAsFields `mapstructure:"tags_as_fields"` + Enabled bool `mapstructure:"-"` + TLS tlscfg.Options `mapstructure:"tls"` + UseReadWriteAliases bool `mapstructure:"use_aliases"` + CreateIndexTemplates bool `mapstructure:"create_mappings"` + UseILM bool `mapstructure:"use_ilm"` + Version uint `mapstructure:"version"` + LogLevel string `mapstructure:"log_level"` } // TagsAsFields holds configuration for tag schema. diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 48b8798af44..88d38105fe1 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -17,7 +17,6 @@ package es import ( "flag" - "fmt" "strings" "time" @@ -29,36 +28,36 @@ import ( ) const ( - suffixUsername = ".username" - suffixPassword = ".password" - suffixSniffer = ".sniffer" - suffixSnifferTLSEnabled = ".sniffer-tls-enabled" - suffixTokenPath = ".token-file" - suffixServerURLs = ".server-urls" - suffixRemoteReadClusters = ".remote-read-clusters" - suffixMaxSpanAge = ".max-span-age" - suffixNumShards = ".num-shards" - suffixNumReplicas = ".num-replicas" - suffixBulkSize = ".bulk.size" - suffixBulkWorkers = ".bulk.workers" - suffixBulkActions = ".bulk.actions" - suffixBulkFlushInterval = ".bulk.flush-interval" - suffixTimeout = ".timeout" - suffixIndexPrefix = ".index-prefix" - suffixIndexDateSeparator = ".index-date-separator" - suffixIndexRollover = ".index-rollover" - suffixTagsAsFields = ".tags-as-fields" - suffixTagsAsFieldsAll = suffixTagsAsFields + ".all" - suffixTagsAsFieldsInclude = suffixTagsAsFields + ".include" - suffixTagsFile = suffixTagsAsFields + ".config-file" - suffixTagDeDotChar = suffixTagsAsFields + ".dot-replacement" - suffixReadAlias = ".use-aliases" - suffixUseILM = ".use-ilm" - suffixCreateIndexTemplate = ".create-index-templates" - suffixEnabled = ".enabled" - suffixVersion = ".version" - suffixMaxDocCount = ".max-doc-count" - suffixLogLevel = ".log-level" + suffixUsername = ".username" + suffixPassword = ".password" + suffixSniffer = ".sniffer" + suffixSnifferTLSEnabled = ".sniffer-tls-enabled" + suffixTokenPath = ".token-file" + suffixServerURLs = ".server-urls" + suffixRemoteReadClusters = ".remote-read-clusters" + suffixMaxSpanAge = ".max-span-age" + suffixNumShards = ".num-shards" + suffixNumReplicas = ".num-replicas" + suffixBulkSize = ".bulk.size" + suffixBulkWorkers = ".bulk.workers" + suffixBulkActions = ".bulk.actions" + suffixBulkFlushInterval = ".bulk.flush-interval" + suffixTimeout = ".timeout" + suffixIndexPrefix = ".index-prefix" + suffixIndexDateSeparator = ".index-date-separator" + suffixIndexRolloverFrequency = ".index-rollover-frequency" + suffixTagsAsFields = ".tags-as-fields" + suffixTagsAsFieldsAll = suffixTagsAsFields + ".all" + suffixTagsAsFieldsInclude = suffixTagsAsFields + ".include" + suffixTagsFile = suffixTagsAsFields + ".config-file" + suffixTagDeDotChar = suffixTagsAsFields + ".dot-replacement" + suffixReadAlias = ".use-aliases" + suffixUseILM = ".use-ilm" + suffixCreateIndexTemplate = ".create-index-templates" + suffixEnabled = ".enabled" + suffixVersion = ".version" + suffixMaxDocCount = ".max-doc-count" + suffixLogLevel = ".log-level" // default number of documents to return from a query (elasticsearch allowed limit) // see search.max_buckets and index.max_result_window defaultMaxDocCount = 10_000 @@ -67,7 +66,7 @@ const ( // default separator for Elasticsearch index date layout. defaultIndexDateSeparator = "-" - defaultIndexRollover = "day" + defaultIndexRolloverFrequency = "day" ) // TODO this should be moved next to config.Configuration struct (maybe ./flags package) @@ -210,8 +209,8 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { defaultIndexDateSeparator, "Optional date separator of Jaeger indices. For example \".\" creates \"jaeger-span-2020.11.20\".") flagSet.String( - nsConfig.namespace+suffixIndexRollover, - defaultIndexRollover, + nsConfig.namespace+suffixIndexRolloverFrequency, + defaultIndexRolloverFrequency, "Rotates Jaeger indices over the given period. For example \"day\" creates \"jaeger-span-yyyy-MM-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ "Jaeger additionally supports manual and automated (via ILM) index management. Reference: https://www.jaegertracing.io/docs/deployment/#elasticsearch-rollover.") flagSet.Bool( @@ -325,9 +324,9 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { cfg.RemoteReadClusters = strings.Split(remoteReadClusters, ",") } - rolloverBy := strings.ToLower(v.GetString(cfg.namespace + suffixIndexRollover)) + rolloverFreq := strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequency)) separator := v.GetString(cfg.namespace + suffixIndexDateSeparator) - cfg.IndexDateLayout = initDateLayout(rolloverBy, separator) + cfg.IndexDateLayout = initDateLayout(rolloverFreq, separator) } // GetPrimary returns primary configuration. @@ -355,11 +354,10 @@ func stripWhiteSpace(str string) string { } func initDateLayout(rolloverFreq, sep string) string { - // default to daily format - indexLayout := "2006" + sep + "01" + sep + "02" - if rolloverFreq == "hour" { - indexLayout = indexLayout + sep + "15" - } - return indexLayout -} + // default to daily format + indexLayout := "2006" + sep + "01" + sep + "02" + if rolloverFreq == "hour" { + indexLayout = indexLayout + sep + "15" + } + return indexLayout } diff --git a/plugin/storage/es/options_test.go b/plugin/storage/es/options_test.go index b98b7c9b865..826442480ac 100644 --- a/plugin/storage/es/options_test.go +++ b/plugin/storage/es/options_test.go @@ -182,8 +182,8 @@ func TestIndexRollover(t *testing.T) { wantDateLayout string }{ {"not defined (default)", []string{}, "2006-01-02"}, - {"day rotation", []string{"--es.index-rollover=day"}, "2006-01-02"}, - {"hour rotation", []string{"--es.index-rollover=hour"}, "2006-01-02-15"}, + {"day rotation", []string{"--es.index-rollover-frequency=day"}, "2006-01-02"}, + {"hour rotation", []string{"--es.index-rollover-frequency=hour"}, "2006-01-02-15"}, {"error rotation change default", []string{"--es.index-rollover=hours"}, "2006-01-02"}, } for _, tc := range testCases { diff --git a/plugin/storage/es/spanstore/reader.go b/plugin/storage/es/spanstore/reader.go index 9c59a5791bf..2ed5d9f3c76 100644 --- a/plugin/storage/es/spanstore/reader.go +++ b/plugin/storage/es/spanstore/reader.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "time" "github.com/olivere/elastic" @@ -199,11 +200,16 @@ func timeRangeIndices(indexName, indexDateLayout string, startTime time.Time, en var indices []string firstIndex := indexWithDate(indexName, indexDateLayout, startTime) currentIndex := indexWithDate(indexName, indexDateLayout, endTime) + + reduce := -24 * time.Hour + if strings.HasSuffix(indexDateLayout, "15") { + reduce = -1 * time.Hour + } for currentIndex != firstIndex { if len(indices) == 0 || indices[len(indices)-1] != currentIndex { indices = append(indices, currentIndex) } - endTime = endTime.Add(-1 * time.Hour) + endTime = endTime.Add(reduce) currentIndex = indexWithDate(indexName, indexDateLayout, endTime) } indices = append(indices, firstIndex) From b79d7a73dfb54d32cd38d2db922864497d7dbb6a Mon Sep 17 00:00:00 2001 From: WalkerWang731 Date: Mon, 26 Apr 2021 14:56:44 +0800 Subject: [PATCH 6/8] update the suggestion from yurishkuro Co-authored-by: Yuri Shkuro Signed-off-by: WalkerWang731 --- plugin/storage/es/options.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 4eb73f36478..48b8798af44 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -354,15 +354,12 @@ func stripWhiteSpace(str string) string { return strings.Replace(str, " ", "", -1) } -func initDateLayout(rolloverBy, separator string) string { - seps := []interface{}{separator, separator} - - // Default "day" index layout. - indexLayout := "2006%s01%s02" - - if rolloverBy == "hour" { - indexLayout = "2006%s01%s02%s15" - seps = append(seps, separator) - } - return fmt.Sprintf(indexLayout, seps...) +func initDateLayout(rolloverFreq, sep string) string { + // default to daily format + indexLayout := "2006" + sep + "01" + sep + "02" + if rolloverFreq == "hour" { + indexLayout = indexLayout + sep + "15" + } + return indexLayout +} } From 7e564d4074ad542f6a32ce6b41d633dda772d4d8 Mon Sep 17 00:00:00 2001 From: WalkerWang731 Date: Tue, 27 Apr 2021 15:15:11 +0800 Subject: [PATCH 7/8] update unit test for the es reader_test Signed-off-by: WalkerWang731 --- plugin/storage/es/spanstore/reader_test.go | 36 ++++++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/plugin/storage/es/spanstore/reader_test.go b/plugin/storage/es/spanstore/reader_test.go index 4668a00ff2d..0862aa3b3bc 100644 --- a/plugin/storage/es/spanstore/reader_test.go +++ b/plugin/storage/es/spanstore/reader_test.go @@ -147,20 +147,28 @@ func TestSpanReaderIndices(t *testing.T) { logger, _ := testutils.NewLogger() metricsFactory := metricstest.NewFactory(0) date := time.Date(2019, 10, 10, 5, 0, 0, 0, time.UTC) - dateFormat := date.UTC().Format("2006-01-02") + dateFormatDay := "2006-01-02" + dateFormatHour := "2006-01-02-15" + testCases := []struct { indices []string params SpanReaderParams }{ {params: SpanReaderParams{Client: client, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "", Archive: false}, - indices: []string{spanIndex + dateFormat}}, + IndexPrefix: "", Archive: false, IndexDateLayout: dateFormatDay}, + indices: []string{spanIndex + date.UTC().Format(dateFormatDay)}}, + {params: SpanReaderParams{Client: client, Logger: logger, MetricsFactory: metricsFactory, + IndexPrefix: "", Archive: false, IndexDateLayout: dateFormatHour}, + indices: []string{spanIndex + date.UTC().Format(dateFormatHour)}}, {params: SpanReaderParams{Client: client, Logger: logger, MetricsFactory: metricsFactory, IndexPrefix: "", UseReadWriteAliases: true}, indices: []string{spanIndex + "read"}}, {params: SpanReaderParams{Client: client, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "foo:", Archive: false}, - indices: []string{"foo:" + indexPrefixSeparator + spanIndex + dateFormat}}, + IndexPrefix: "foo:", Archive: false, IndexDateLayout: dateFormatDay}, + indices: []string{"foo:" + indexPrefixSeparator + spanIndex + date.UTC().Format(dateFormatDay)}}, + {params: SpanReaderParams{Client: client, Logger: logger, MetricsFactory: metricsFactory, + IndexPrefix: "foo:", Archive: false, IndexDateLayout: dateFormatHour}, + indices: []string{"foo:" + indexPrefixSeparator + spanIndex + date.UTC().Format(dateFormatHour)}}, {params: SpanReaderParams{Client: client, Logger: logger, MetricsFactory: metricsFactory, IndexPrefix: "foo:", UseReadWriteAliases: true}, indices: []string{"foo:-" + spanIndex + "read"}}, @@ -174,11 +182,19 @@ func TestSpanReaderIndices(t *testing.T) { IndexPrefix: "foo:", Archive: true, UseReadWriteAliases: true}, indices: []string{"foo:" + indexPrefixSeparator + spanIndex + archiveReadIndexSuffix}}, {params: SpanReaderParams{Client: client, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "", Archive: false, RemoteReadClusters: []string{"cluster_one", "cluster_two"}}, + IndexPrefix: "", Archive: false, RemoteReadClusters: []string{"cluster_one", "cluster_two"}, + IndexDateLayout: dateFormatDay}, + indices: []string{ + spanIndex + date.UTC().Format(dateFormatDay), + "cluster_one:" + spanIndex + date.UTC().Format(dateFormatDay), + "cluster_two:" + spanIndex + date.UTC().Format(dateFormatDay)}}, + {params: SpanReaderParams{Client: client, Logger: logger, MetricsFactory: metricsFactory, + IndexPrefix: "", Archive: false, RemoteReadClusters: []string{"cluster_one", "cluster_two"}, + IndexDateLayout: dateFormatHour}, indices: []string{ - spanIndex + dateFormat, - "cluster_one:" + spanIndex + dateFormat, - "cluster_two:" + spanIndex + dateFormat}}, + spanIndex + date.UTC().Format(dateFormatHour), + "cluster_one:" + spanIndex + date.UTC().Format(dateFormatHour), + "cluster_two:" + spanIndex + date.UTC().Format(dateFormatHour)}}, {params: SpanReaderParams{Client: client, Logger: logger, MetricsFactory: metricsFactory, IndexPrefix: "", Archive: true, RemoteReadClusters: []string{"cluster_one", "cluster_two"}}, indices: []string{ @@ -200,7 +216,7 @@ func TestSpanReaderIndices(t *testing.T) { } for _, testCase := range testCases { r := NewSpanReader(testCase.params) - actual := r.timeRangeIndices(r.spanIndexPrefix, "2006-01-02", date, date) + actual := r.timeRangeIndices(r.spanIndexPrefix, r.indexDateLayout, date, date) assert.Equal(t, testCase.indices, actual) } } From 5824c2eb2b6c0c4499c346687823dd1ea4c222a1 Mon Sep 17 00:00:00 2001 From: WalkerWang731 Date: Mon, 26 Apr 2021 14:56:44 +0800 Subject: [PATCH 8/8] update the suggestion from yurishkuro Co-authored-by: Yuri Shkuro Signed-off-by: WalkerWang731 --- plugin/storage/es/options.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 4eb73f36478..48b8798af44 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -354,15 +354,12 @@ func stripWhiteSpace(str string) string { return strings.Replace(str, " ", "", -1) } -func initDateLayout(rolloverBy, separator string) string { - seps := []interface{}{separator, separator} - - // Default "day" index layout. - indexLayout := "2006%s01%s02" - - if rolloverBy == "hour" { - indexLayout = "2006%s01%s02%s15" - seps = append(seps, separator) - } - return fmt.Sprintf(indexLayout, seps...) +func initDateLayout(rolloverFreq, sep string) string { + // default to daily format + indexLayout := "2006" + sep + "01" + sep + "02" + if rolloverFreq == "hour" { + indexLayout = indexLayout + sep + "15" + } + return indexLayout +} }