From 767df00bb3a2cc155d4525c583a7cd38d057fb6d Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Thu, 24 Jun 2021 11:07:04 +0200 Subject: [PATCH 1/8] Add profile. Signed-off-by: Bartlomiej Plotka --- go.mod | 1 + go.sum | 9 +++++++++ main.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/go.mod b/go.mod index 1573305..63e9207 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/charmbracelet/glamour v0.3.0 github.com/efficientgo/tools/core v0.0.0-20210609125236-d73259166f20 github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20 + github.com/felixge/fgprof v0.9.1 // indirect github.com/go-kit/kit v0.10.0 github.com/gobwas/glob v0.2.3 github.com/gocolly/colly/v2 v2.1.1-0.20201013153555-8252c346cfb0 diff --git a/go.sum b/go.sum index b4dd9e2..ff6be85 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,9 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/charmbracelet/glamour v0.3.0 h1:3H+ZrKlSg8s+WU6V7eF2eRVYt8lCueffbi7r2+ffGkc= github.com/charmbracelet/glamour v0.3.0/go.mod h1:TzF0koPZhqq0YVBNL100cPHznAAjVj7fksX2RInwjGw= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -151,6 +154,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanw/esbuild v0.6.5/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/fgprof v0.9.1 h1:E6FUJ2Mlv043ipLOCFqo8+cHo9MhQ203E2cdEK/isEs= +github.com/felixge/fgprof v0.9.1/go.mod h1:7/HK6JFtFaARhIljgP2IV8rJLIoHDoOYoUphsnGvqxE= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -220,6 +225,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040 h1:i7RUpu0EybzQyQvPT7J3MmODs4+gPcHsD/pqW0uIYVo= +github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -267,6 +274,7 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= @@ -616,6 +624,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/main.go b/main.go index 40ee609..ec41919 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,10 @@ import ( "github.com/bwplotka/mdox/pkg/transform" "github.com/bwplotka/mdox/pkg/version" "github.com/charmbracelet/glamour" + "github.com/efficientgo/tools/core/pkg/errcapture" + "github.com/efficientgo/tools/core/pkg/logerrcapture" extflag "github.com/efficientgo/tools/extkingpin" + "github.com/felixge/fgprof" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -66,6 +69,7 @@ func main() { Default("info").Enum("error", "warn", "info", "debug") logFormat := app.Flag("log.format", "Log format to use."). Default(logFormatCLILog).Enum(logFormatLogfmt, logFormatJson, logFormatCLILog) + profilesPath := app.Flag("debug.profiles", "WHAT THE HECK ARE YOU DOING SO LONG").Hidden().String() ctx, cancel := context.WithCancel(context.Background()) registerFmt(ctx, app) @@ -74,6 +78,15 @@ func main() { cmd, runner := app.Parse() logger := setupLogger(*logLevel, *logFormat) + if *profilesPath != "" { + finalize, err := snapshotProfiles(*profilesPath) + if err != nil { + level.Error(logger).Log("err", errors.Wrapf(err, "%s profiles init failed", cmd)) + os.Exit(1) + } + defer logerrcapture.Do(logger, finalize, "profiles") + } + var g run.Group g.Add(func() error { // TODO(bwplotka): Move to customized better setup function. @@ -103,6 +116,23 @@ func main() { } } +func snapshotProfiles(dir string) (func() error, error) { + // TODO: now -> date + if err := os.MkdirAll(filepath.Join(dir, "now"), os.ModePerm); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.Join(dir, "now", "fgprof.pprof"), os.O_TRUNC|os.O_WRONLY, os.ModePerm) + if err != nil { + return nil, err + } + + fgFunc := fgprof.Start(f, fgprof.FormatPprof) + + return func() (err error) { + defer errcapture.Do(&err, f.Close, "close") + return fgFunc() + }, nil +} func interrupt(logger log.Logger, cancel <-chan struct{}) error { c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) From 77031977b159171b9caccfc36e588c0f258b2518 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Thu, 8 Jul 2021 11:32:07 +0530 Subject: [PATCH 2/8] Add metrics and memory profile Signed-off-by: Saswata Mukherjee --- go.mod | 3 +- go.sum | 8 ++ main.go | 52 ++++++++++-- pkg/mdformatter/linktransformer/link.go | 86 +++++++++++++++++++- pkg/mdformatter/linktransformer/link_test.go | 68 ++++++++-------- pkg/mdformatter/linktransformer/validator.go | 10 +++ pkg/mdformatter/mdformatter.go | 45 ++++++++-- pkg/mdformatter/mdformatter_test.go | 4 +- pkg/transform/transform.go | 4 +- 9 files changed, 225 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 63e9207..de2a0f3 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/charmbracelet/glamour v0.3.0 github.com/efficientgo/tools/core v0.0.0-20210609125236-d73259166f20 github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20 - github.com/felixge/fgprof v0.9.1 // indirect + github.com/felixge/fgprof v0.9.1 github.com/go-kit/kit v0.10.0 github.com/gobwas/glob v0.2.3 github.com/gocolly/colly/v2 v2.1.1-0.20201013153555-8252c346cfb0 @@ -21,6 +21,7 @@ require ( github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 // indirect github.com/oklog/run v1.1.0 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.3.0 github.com/sergi/go-diff v1.0.0 github.com/theckman/yacspin v0.8.0 github.com/yuin/goldmark v1.3.5 diff --git a/go.sum b/go.sum index ff6be85..ac3b128 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,7 @@ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuP github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= @@ -101,7 +102,9 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/glamour v0.3.0 h1:3H+ZrKlSg8s+WU6V7eF2eRVYt8lCueffbi7r2+ffGkc= github.com/charmbracelet/glamour v0.3.0/go.mod h1:TzF0koPZhqq0YVBNL100cPHznAAjVj7fksX2RInwjGw= @@ -331,6 +334,7 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.6 h1:ZOvqHKtnx0fUpnbQm3m3zKFWE+DRC+XB1onh8JoEObE= github.com/microcosm-cc/bluemonday v1.0.6/go.mod h1:HOT/6NaBlR0f9XlxD3zolN6Z3N8Lp4pvhp+jLS5ihnI= @@ -412,21 +416,25 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= diff --git a/main.go b/main.go index ec41919..0eafe87 100644 --- a/main.go +++ b/main.go @@ -6,9 +6,12 @@ package main import ( "context" "fmt" + "net/http" "os" "os/signal" "path/filepath" + "runtime" + "runtime/pprof" "strings" "syscall" @@ -28,6 +31,8 @@ import ( "github.com/go-kit/kit/log/level" "github.com/oklog/run" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "gopkg.in/alecthomas/kingpin.v2" ) @@ -37,6 +42,10 @@ const ( logFormatCLILog = "clilog" ) +type mdoxMetrics struct { + reg *prometheus.Registry +} + func setupLogger(logLevel, logFormat string) log.Logger { var lvl level.Option switch logLevel { @@ -69,10 +78,14 @@ func main() { Default("info").Enum("error", "warn", "info", "debug") logFormat := app.Flag("log.format", "Log format to use."). Default(logFormatCLILog).Enum(logFormatLogfmt, logFormatJson, logFormatCLILog) - profilesPath := app.Flag("debug.profiles", "WHAT THE HECK ARE YOU DOING SO LONG").Hidden().String() + // Profiling and metrics. + profilesPath := app.Flag("debug.profiles", "Path to which CPU and heap profiles are saved").Hidden().String() + metrics := app.Flag("metrics", "Enable metrics and view them at https://localhost:9091/metrics").Hidden().Bool() + + m := &mdoxMetrics{reg: nil} ctx, cancel := context.WithCancel(context.Background()) - registerFmt(ctx, app) + registerFmt(ctx, app, m) registerTransform(ctx, app) cmd, runner := app.Parse() @@ -95,6 +108,19 @@ func main() { cancel() }) + if *metrics { + srv := &http.Server{Addr: ":9091"} + m.reg = prometheus.NewRegistry() + + g.Add(func() error { + http.Handle("/metrics", promhttp.HandlerFor(m.reg, promhttp.HandlerOpts{})) + return srv.ListenAndServe() + }, func(err error) { + _ = srv.Shutdown(ctx) + cancel() + }) + } + // Listen for termination signals. { cancel := make(chan struct{}) @@ -114,6 +140,7 @@ func main() { level.Error(logger).Log("err", errors.Wrapf(err, "%s command failed", cmd)) os.Exit(1) } + } func snapshotProfiles(dir string) (func() error, error) { @@ -121,11 +148,21 @@ func snapshotProfiles(dir string) (func() error, error) { if err := os.MkdirAll(filepath.Join(dir, "now"), os.ModePerm); err != nil { return nil, err } - f, err := os.OpenFile(filepath.Join(dir, "now", "fgprof.pprof"), os.O_TRUNC|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(filepath.Join(dir, "now", "fgprof.pb.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) if err != nil { return nil, err } + m, err := os.OpenFile(filepath.Join(dir, "now", "memprof.pb.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) + if err != nil { + return nil, err + } + runtime.GC() + + if err := pprof.WriteHeapProfile(m); err != nil { + return nil, err + } + fgFunc := fgprof.Start(f, fgprof.FormatPprof) return func() (err error) { @@ -133,6 +170,7 @@ func snapshotProfiles(dir string) (func() error, error) { return fgFunc() }, nil } + func interrupt(logger log.Logger, cancel <-chan struct{}) error { c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) @@ -145,7 +183,7 @@ func interrupt(logger log.Logger, cancel <-chan struct{}) error { } } -func registerFmt(_ context.Context, app *extkingpin.App) { +func registerFmt(_ context.Context, app *extkingpin.App, m *mdoxMetrics) { cmd := app.Command("fmt", "Formats in-place given markdown files uniformly following GFM (Github Flavored Markdown: https://github.github.com/gfm/). Example: mdox fmt *.md") files := cmd.Arg("files", "Markdown file(s) to process.").Required().ExistingFiles() checkOnly := cmd.Flag("check", "If true, fmt will not modify the given files, instead it will fail if files needs formatting").Bool() @@ -187,7 +225,7 @@ This directive runs executable with arguments and put its stderr and stdout outp if err != nil { return err } - v, err := linktransformer.NewValidator(ctx, logger, validateConfigContent, anchorDir) + v, err := linktransformer.NewValidator(ctx, logger, validateConfigContent, anchorDir, m.reg) if err != nil { return err } @@ -202,7 +240,7 @@ This directive runs executable with arguments and put its stderr and stdout outp } if *checkOnly { - diff, err := mdformatter.IsFormatted(ctx, logger, *files, opts...) + diff, err := mdformatter.IsFormatted(ctx, logger, *files, m.reg, opts...) if err != nil { return err } @@ -223,7 +261,7 @@ This directive runs executable with arguments and put its stderr and stdout outp return errors.Errorf("files not formatted: %v", diffOut) } - return mdformatter.Format(ctx, logger, *files, opts...) + return mdformatter.Format(ctx, logger, *files, m.reg, opts...) }) } diff --git a/pkg/mdformatter/linktransformer/link.go b/pkg/mdformatter/linktransformer/link.go index 2f6044f..690d88f 100644 --- a/pkg/mdformatter/linktransformer/link.go +++ b/pkg/mdformatter/linktransformer/link.go @@ -9,6 +9,7 @@ import ( "context" "io" "net/http" + "net/url" "os" "path/filepath" "regexp" @@ -24,6 +25,8 @@ import ( "github.com/go-kit/kit/log/level" "github.com/gocolly/colly/v2" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) var remoteLinkPrefixRe = regexp.MustCompile(`^http[s]?://`) @@ -35,6 +38,54 @@ var ( IDNotFoundErr = LookupError(errors.New("file exists, but does not have such id")) ) +type linktransformerMetrics struct { + localLinksChecked prometheus.Counter + remoteLinksChecked prometheus.Counter + roundTripLinks prometheus.Counter + githubSkippedLinks prometheus.Counter + ignoreSkippedLinks prometheus.Counter + + collyRequests *prometheus.CounterVec + collyPerDomainLatency *prometheus.HistogramVec +} + +func newLinktransformerMetrics(reg *prometheus.Registry) *linktransformerMetrics { + l := &linktransformerMetrics{} + + l.localLinksChecked = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "mdox_local_links_total", + Help: "The total number of local links which were checked", + }) + l.remoteLinksChecked = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "mdox_remote_links_total", + Help: "The total number of remote links which were checked", + }) + l.roundTripLinks = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "mdox_round_trip_links_total", + Help: "The total number of links which were roundtrip checked", + }) + l.githubSkippedLinks = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "mdox_github_skipped_links_total", + Help: "The total number of links which were github checked", + }) + l.ignoreSkippedLinks = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "mdox_ignore_skipped_links_total", + Help: "The total number of links which were ignore checked", + }) + + l.collyRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "mdox_colly_requests_total"}, + []string{}, + ) + l.collyPerDomainLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{Name: "mdox_colly_per_domain_latency", Buckets: prometheus.DefBuckets}, + []string{"domain"}, + ) + + reg.MustRegister(l.localLinksChecked, l.remoteLinksChecked, l.roundTripLinks, l.githubSkippedLinks, l.ignoreSkippedLinks, l.collyRequests, l.collyPerDomainLatency) + return l +} + const ( originalURLKey = "originalURLKey" numberOfRetriesKey = "retryKey" @@ -126,6 +177,9 @@ type validator struct { futureMu sync.Mutex destFutures map[futureKey]*futureResult + + l *linktransformerMetrics + transportFn func(url string) http.RoundTripper } type futureKey struct { @@ -140,7 +194,7 @@ type futureResult struct { // NewValidator returns mdformatter.LinkTransformer that crawls all links. // TODO(bwplotka): Add optimization and debug modes - this is the main source of latency and pain. -func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []byte, anchorDir string) (mdformatter.LinkTransformer, error) { +func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []byte, anchorDir string, reg *prometheus.Registry) (mdformatter.LinkTransformer, error) { var err error config := Config{} if string(linksValidateConfig) != "" { @@ -149,6 +203,7 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig [] return nil, err } } + v := &validator{ logger: logger, anchorDir: anchorDir, @@ -157,7 +212,26 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig [] remoteLinks: map[string]error{}, c: colly.NewCollector(colly.Async(), colly.StdlibContext(ctx)), destFutures: map[futureKey]*futureResult{}, + l: nil, + transportFn: func(url string) http.RoundTripper { + return http.DefaultTransport + }, + } + + if reg != nil { + v.l = newLinktransformerMetrics(reg) + v.transportFn = func(u string) http.RoundTripper { + parsed, err := url.Parse(u) + if err != nil { + panic(err) + } + return promhttp.InstrumentRoundTripperCounter( + v.l.collyRequests, + promhttp.InstrumentRoundTripperDuration(v.l.collyPerDomainLatency.MustCurryWith(prometheus.Labels{"domain": parsed.Host}), http.DefaultTransport), + ) + } } + // Set very soft limits. // E.g github has 50-5000 https://docs.github.com/en/free-pro-team@latest/rest/reference/rate-limit limit depending // on api (only search is below 100). @@ -225,8 +299,8 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig [] } // MustNewValidator returns mdformatter.LinkTransformer that crawls all links. -func MustNewValidator(logger log.Logger, linksValidateConfig []byte, anchorDir string) mdformatter.LinkTransformer { - v, err := NewValidator(context.TODO(), logger, linksValidateConfig, anchorDir) +func MustNewValidator(logger log.Logger, linksValidateConfig []byte, anchorDir string, reg *prometheus.Registry) mdformatter.LinkTransformer { + v, err := NewValidator(context.TODO(), logger, linksValidateConfig, anchorDir, reg) if err != nil { panic(err) } @@ -286,6 +360,9 @@ func (v *validator) visit(filepath string, dest string, lineNumbers string) { v.destFutures[k] = &futureResult{cases: 1, resultFn: func() error { return nil }} matches := remoteLinkPrefixRe.FindAllStringIndex(dest, 1) if matches == nil { + if v.l != nil { + v.l.localLinksChecked.Inc() + } // Relative or absolute path. Check if exists. newDest := absLocalLink(v.anchorDir, filepath, dest) @@ -295,6 +372,9 @@ func (v *validator) visit(filepath string, dest string, lineNumbers string) { } return } + if v.l != nil { + v.l.remoteLinksChecked.Inc() + } validator := v.validateConfig.GetValidatorForURL(dest) if validator != nil { matched, err := validator.IsValid(k, v) diff --git a/pkg/mdformatter/linktransformer/link_test.go b/pkg/mdformatter/linktransformer/link_test.go index 7407c8c..37eea51 100644 --- a/pkg/mdformatter/linktransformer/link_test.go +++ b/pkg/mdformatter/linktransformer/link_test.go @@ -51,13 +51,13 @@ func TestLocalizer_TransformDestination(t *testing.T) { diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{ filepath.Join(tmpDir, "repo", "docs", "a", "doc.md"), filepath.Join(tmpDir, "repo", "docs", "doc2.md"), - }) + }, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) }) t.Run("no domain specified", func(t *testing.T) { - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, mdformatter.WithLinkTransformer( + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, nil, mdformatter.WithLinkTransformer( NewLocalizer(logger, regexp.MustCompile(`^$`), anchorDir), )) testutil.Ok(t, err) @@ -75,7 +75,7 @@ func TestLocalizer_TransformDestination(t *testing.T) { }) t.Run("domain specified, but without full path", func(t *testing.T) { - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, mdformatter.WithLinkTransformer( + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, nil, mdformatter.WithLinkTransformer( NewLocalizer(logger, regexp.MustCompile(`myproject.example.com`), anchorDir), )) testutil.Ok(t, err) @@ -93,7 +93,7 @@ func TestLocalizer_TransformDestination(t *testing.T) { }) t.Run("domain specified", func(t *testing.T) { - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, mdformatter.WithLinkTransformer( + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, nil, mdformatter.WithLinkTransformer( NewLocalizer(logger, regexp.MustCompile(`myproject.example.com/tip/`), anchorDir), )) testutil.Ok(t, err) @@ -135,12 +135,12 @@ func TestValidator_TransformDestination(t *testing.T) { testFile := filepath.Join(tmpDir, "repo", "docs", "test", "valid-link.md") testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://bwplotka.dev/about\n"), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir), + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte(""), anchorDir, nil), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -152,12 +152,12 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://bwplotka.dev/about\n"), os.ModePerm)) testutil.Ok(t, ioutil.WriteFile(testFile2, []byte("https://bwplotka.dev/about\n"), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir), + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte(""), anchorDir, nil), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -170,12 +170,12 @@ func TestValidator_TransformDestination(t *testing.T) { [1](.) [2](#yolo) [3](../test/valid-local-links.md) [4](../test/valid-local-links.md#yolo) [5](../a/doc.md) `), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir), + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte(""), anchorDir, nil), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -192,12 +192,12 @@ func TestValidator_TransformDestination(t *testing.T) { [2](#run-time-deduplication-of-ha-groups) `), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir), + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte(""), anchorDir, nil), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -214,12 +214,12 @@ func TestValidator_TransformDestination(t *testing.T) { [2](#hugo-का-उपयोग-करते-हुए-स्थानीय-रूप-से-साइट-चलाना) `), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir), + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte(""), anchorDir, nil), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -237,12 +237,12 @@ func TestValidator_TransformDestination(t *testing.T) { [1](.) [2](#not-yolo) [3](../test2/invalid-local-links.md) [4](../test/invalid-local-links.md#not-yolo) [5](../test/doc.md) `), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir), + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte(""), anchorDir, nil), )) testutil.NotOk(t, err) @@ -263,12 +263,12 @@ func TestValidator_TransformDestination(t *testing.T) { relDirPath, err := filepath.Rel(wdir, tmpDir) testutil.Ok(t, err) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir), + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte(""), anchorDir, nil), )) testutil.NotOk(t, err) testutil.Equals(t, fmt.Sprintf("%v%v: %v%v:1: \"https://bwplotka.dev/does-not-exists\" not accessible; status code 404: Not Found", tmpDir, filePath, relDirPath, filePath), err.Error()) @@ -278,12 +278,12 @@ func TestValidator_TransformDestination(t *testing.T) { testFile := filepath.Join(tmpDir, "repo", "docs", "test", "invalid-link2.md") testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://www.github.com/ https://bwplotka.dev/does-not-exits\n"), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka'\n type: 'ignore'\n"), anchorDir), + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka'\n type: 'ignore'\n"), anchorDir, nil), )) testutil.Ok(t, err) }) @@ -292,12 +292,12 @@ func TestValidator_TransformDestination(t *testing.T) { testFile := filepath.Join(tmpDir, "repo", "docs", "test", "links.md") testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://fakelink1.com/ http://fakelink2.com/ https://www.fakelink3.com/\n"), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: '(^http[s]?:\\/\\/)(www\\.)?(fakelink[0-9]\\.com\\/)'\n type: 'ignore'\n"), anchorDir), + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: '(^http[s]?:\\/\\/)(www\\.)?(fakelink[0-9]\\.com\\/)'\n type: 'ignore'\n"), anchorDir, nil), )) testutil.Ok(t, err) }) @@ -308,12 +308,12 @@ func TestValidator_TransformDestination(t *testing.T) { // This is substituted in config using PathorContent flag. But need to pass it directly here. repoToken := os.Getenv("GITHUB_TOKEN") - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka\\/mdox'\n token: '"+repoToken+"'\n type: 'github'\n"), anchorDir), + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka\\/mdox'\n token: '"+repoToken+"'\n type: 'github'\n"), anchorDir, nil), )) testutil.Ok(t, err) }) diff --git a/pkg/mdformatter/linktransformer/validator.go b/pkg/mdformatter/linktransformer/validator.go index 1000813..f86ab57 100644 --- a/pkg/mdformatter/linktransformer/validator.go +++ b/pkg/mdformatter/linktransformer/validator.go @@ -18,6 +18,9 @@ type Validator interface { func (v GitHubValidator) IsValid(k futureKey, r *validator) (bool, error) { // Find rightmost index of match i.e, where regex match ends. // This will be where issue/PR number starts. Split incase of section link and convert to int. + if r.l != nil { + r.l.githubSkippedLinks.Inc() + } rightmostIndex := v._regex.FindStringIndex(k.dest) stringNum := strings.Split(k.dest[rightmostIndex[1]:], "#") num, err := strconv.Atoi(stringNum[0]) @@ -34,6 +37,9 @@ func (v GitHubValidator) IsValid(k futureKey, r *validator) (bool, error) { // RoundTripValidator.IsValid returns true if url is checked by colly. func (v RoundTripValidator) IsValid(k futureKey, r *validator) (bool, error) { // Result will be in future. + if r.l != nil { + r.l.roundTripLinks.Inc() + } r.destFutures[k].resultFn = func() error { return r.remoteLinks[k.dest] } r.rMu.RLock() if _, ok := r.remoteLinks[k.dest]; ok { @@ -49,6 +55,7 @@ func (v RoundTripValidator) IsValid(k futureKey, r *validator) (bool, error) { return true, nil } + r.c.WithTransport(r.transportFn(k.dest)) if err := r.c.Visit(k.dest); err != nil { r.remoteLinks[k.dest] = errors.Wrapf(err, "remote link %v", k.dest) return false, nil @@ -58,6 +65,9 @@ func (v RoundTripValidator) IsValid(k futureKey, r *validator) (bool, error) { // IgnoreValidator.IsValid returns true if matched so that link in not checked. func (v IgnoreValidator) IsValid(k futureKey, r *validator) (bool, error) { + if r.l != nil { + r.l.ignoreSkippedLinks.Inc() + } return true, nil } diff --git a/pkg/mdformatter/mdformatter.go b/pkg/mdformatter/mdformatter.go index 71793f3..08c9e98 100644 --- a/pkg/mdformatter/mdformatter.go +++ b/pkg/mdformatter/mdformatter.go @@ -20,6 +20,7 @@ import ( "github.com/gohugoio/hugo/parser/pageparser" "github.com/mattn/go-isatty" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/theckman/yacspin" "github.com/yuin/goldmark" "github.com/yuin/goldmark/extension" @@ -27,6 +28,26 @@ import ( "gopkg.in/yaml.v3" ) +type mdformatterMetrics struct { + filesProcessed prometheus.Counter + perFileLatency *prometheus.HistogramVec +} + +func newMdformatterMetrics(reg *prometheus.Registry) *mdformatterMetrics { + m := &mdformatterMetrics{} + m.filesProcessed = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "mdox_processed_files_total", + Help: "The total number of processed files", + }) + m.perFileLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{Name: "mdox_per_file_latency", Buckets: prometheus.DefBuckets}, + []string{"filepath"}, + ) + + reg.MustRegister(m.filesProcessed, m.perFileLatency) + return m +} + type SourceContext struct { context.Context @@ -216,33 +237,36 @@ func newSpinner(suffix string) (*yacspin.Spinner, error) { } // Format formats given markdown files in-place. IsFormatted `With...` function to see what modifiers you can add. -func Format(ctx context.Context, logger log.Logger, files []string, opts ...Option) error { +func Format(ctx context.Context, logger log.Logger, files []string, reg *prometheus.Registry, opts ...Option) error { spin, err := newSpinner(" Formatting: ") if err != nil { return err } - return format(ctx, logger, files, nil, spin, opts...) + return format(ctx, logger, files, nil, spin, reg, opts...) } // IsFormatted tries to formats given markdown files and return Diff if files are not formatted. // If diff is empty it means all files are formatted. -func IsFormatted(ctx context.Context, logger log.Logger, files []string, opts ...Option) (diffs Diffs, err error) { +func IsFormatted(ctx context.Context, logger log.Logger, files []string, reg *prometheus.Registry, opts ...Option) (diffs Diffs, err error) { d := Diffs{} spin, err := newSpinner(" Checking: ") if err != nil { return nil, err } - if err := format(ctx, logger, files, &d, spin, opts...); err != nil { + if err := format(ctx, logger, files, &d, spin, reg, opts...); err != nil { return nil, err } return d, nil } -func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs, spin *yacspin.Spinner, opts ...Option) error { +func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs, spin *yacspin.Spinner, reg *prometheus.Registry, opts ...Option) error { f := New(ctx, opts...) b := bytes.Buffer{} // TODO(bwplotka): Add concurrency (collector will need to redone). - + var m *mdformatterMetrics + if reg != nil { + m = newMdformatterMetrics(reg) + } errs := merrors.New() if spin != nil { errs.Add(spin.Start()) @@ -257,6 +281,11 @@ func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs spin.Message(fn + "...") } errs.Add(func() error { + var start_time time.Time + if reg != nil { + start_time = time.Now() + m.filesProcessed.Inc() + } file, err := os.OpenFile(fn, os.O_RDWR, 0) if err != nil { return errors.Wrapf(err, "open %v", fn) @@ -288,6 +317,10 @@ func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs if err != nil { return errors.Wrapf(err, "write %v", fn) } + if reg != nil { + time_taken := time.Since(start_time) + m.perFileLatency.WithLabelValues(fn).Observe(time_taken.Seconds()) + } return file.Truncate(int64(n)) }()) } diff --git a/pkg/mdformatter/mdformatter_test.go b/pkg/mdformatter/mdformatter_test.go index 1939fb6..0c5f809 100644 --- a/pkg/mdformatter/mdformatter_test.go +++ b/pkg/mdformatter/mdformatter_test.go @@ -42,12 +42,12 @@ func TestFormat_FormatSingle_NoTransformers(t *testing.T) { } func TestCheck_NoTransformers(t *testing.T) { - diff, err := IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/formatted.md"}) + diff, err := IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/formatted.md"}, nil) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff)) testutil.Equals(t, "files the same; no diff", diff.String()) - diff, err = IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/not_formatted.md"}) + diff, err = IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/not_formatted.md"}, nil) testutil.Ok(t, err) exp, err := ioutil.ReadFile("testdata/not_formatted.md.diff") diff --git a/pkg/transform/transform.go b/pkg/transform/transform.go index aa0762d..6e37c2e 100644 --- a/pkg/transform/transform.go +++ b/pkg/transform/transform.go @@ -101,7 +101,7 @@ func Dir(ctx context.Context, logger log.Logger, config []byte) error { } // Once we did all the changes, change links. - return mdformatter.Format(ctx, logger, tr.filesToLinkAdjust, mdformatter.WithLinkTransformer(tr.linkTransformer)) + return mdformatter.Format(ctx, logger, tr.filesToLinkAdjust, nil, mdformatter.WithLinkTransformer(tr.linkTransformer)) } type transformer struct { @@ -217,7 +217,7 @@ func (t *transformer) transformFile(path string, info os.FileInfo, err error) er return nil } - return mdformatter.Format(t.ctx, t.logger, []string{target}, opts...) + return mdformatter.Format(t.ctx, t.logger, []string{target}, nil, opts...) } type relLinkTransformer struct { From 882dcfb99dbd9bb56141d36f5dec7fed1edcdfba Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Sat, 17 Jul 2021 18:18:12 +0530 Subject: [PATCH 3/8] Print metrics Signed-off-by: Saswata Mukherjee --- go.mod | 1 + main.go | 32 +++++++++++++--- pkg/mdformatter/linktransformer/link.go | 40 ++++++++++---------- pkg/mdformatter/linktransformer/link_test.go | 20 +++++----- pkg/mdformatter/linktransformer/validator.go | 14 +++---- pkg/mdformatter/mdformatter.go | 25 ++++++------ 6 files changed, 73 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index de2a0f3..99eb167 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/oklog/run v1.1.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.3.0 + github.com/prometheus/common v0.7.0 github.com/sergi/go-diff v1.0.0 github.com/theckman/yacspin v0.8.0 github.com/yuin/goldmark v1.3.5 diff --git a/main.go b/main.go index 0eafe87..d9f47d8 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "runtime/pprof" "strings" "syscall" + "time" "github.com/bwplotka/mdox/pkg/clilog" "github.com/bwplotka/mdox/pkg/extkingpin" @@ -33,6 +34,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/expfmt" "gopkg.in/alecthomas/kingpin.v2" ) @@ -82,7 +84,7 @@ func main() { profilesPath := app.Flag("debug.profiles", "Path to which CPU and heap profiles are saved").Hidden().String() metrics := app.Flag("metrics", "Enable metrics and view them at https://localhost:9091/metrics").Hidden().Bool() - m := &mdoxMetrics{reg: nil} + m := &mdoxMetrics{} ctx, cancel := context.WithCancel(context.Background()) registerFmt(ctx, app, m) @@ -144,16 +146,16 @@ func main() { } func snapshotProfiles(dir string) (func() error, error) { - // TODO: now -> date - if err := os.MkdirAll(filepath.Join(dir, "now"), os.ModePerm); err != nil { + now := time.Now().UTC() + if err := os.MkdirAll(filepath.Join(dir, strings.ReplaceAll(now.Format(time.UnixDate), " ", "_")), os.ModePerm); err != nil { return nil, err } - f, err := os.OpenFile(filepath.Join(dir, "now", "fgprof.pb.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(filepath.Join(dir, strings.ReplaceAll(now.Format(time.UnixDate), " ", "_"), "fgprof.pb.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) if err != nil { return nil, err } - m, err := os.OpenFile(filepath.Join(dir, "now", "memprof.pb.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) + m, err := os.OpenFile(filepath.Join(dir, strings.ReplaceAll(now.Format(time.UnixDate), " ", "_"), "memprof.pb.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) if err != nil { return nil, err } @@ -171,6 +173,22 @@ func snapshotProfiles(dir string) (func() error, error) { }, nil } +func (m *mdoxMetrics) Print() error { + mfs, err := m.reg.Gather() + if err != nil { + return err + } + + enc := expfmt.NewEncoder(os.Stdout, expfmt.FmtProtoText) + + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + return err + } + } + return nil +} + func interrupt(logger log.Logger, cancel <-chan struct{}) error { c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) @@ -199,6 +217,10 @@ This directive runs executable with arguments and put its stderr and stdout outp linksValidateConfig := extflag.RegisterPathOrContent(cmd, "links.validate.config", "YAML file for skipping link check, with spec defined in github.com/bwplotka/mdox/pkg/linktransformer.ValidatorConfig", extflag.WithEnvSubstitution()) cmd.Run(func(ctx context.Context, logger log.Logger) (err error) { + if m.reg != nil { + defer logerrcapture.Do(logger, m.Print, "print") + } + var opts []mdformatter.Option if !*disableGenCodeBlocksDirectives { opts = append(opts, mdformatter.WithCodeBlockTransformer(mdgen.NewCodeBlockTransformer())) diff --git a/pkg/mdformatter/linktransformer/link.go b/pkg/mdformatter/linktransformer/link.go index 690d88f..d02aa84 100644 --- a/pkg/mdformatter/linktransformer/link.go +++ b/pkg/mdformatter/linktransformer/link.go @@ -82,7 +82,9 @@ func newLinktransformerMetrics(reg *prometheus.Registry) *linktransformerMetrics []string{"domain"}, ) - reg.MustRegister(l.localLinksChecked, l.remoteLinksChecked, l.roundTripLinks, l.githubSkippedLinks, l.ignoreSkippedLinks, l.collyRequests, l.collyPerDomainLatency) + if reg != nil { + reg.MustRegister(l.localLinksChecked, l.remoteLinksChecked, l.roundTripLinks, l.githubSkippedLinks, l.ignoreSkippedLinks, l.collyRequests, l.collyPerDomainLatency) + } return l } @@ -212,24 +214,22 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig [] remoteLinks: map[string]error{}, c: colly.NewCollector(colly.Async(), colly.StdlibContext(ctx)), destFutures: map[futureKey]*futureResult{}, - l: nil, + l: &linktransformerMetrics{}, transportFn: func(url string) http.RoundTripper { return http.DefaultTransport }, } - if reg != nil { - v.l = newLinktransformerMetrics(reg) - v.transportFn = func(u string) http.RoundTripper { - parsed, err := url.Parse(u) - if err != nil { - panic(err) - } - return promhttp.InstrumentRoundTripperCounter( - v.l.collyRequests, - promhttp.InstrumentRoundTripperDuration(v.l.collyPerDomainLatency.MustCurryWith(prometheus.Labels{"domain": parsed.Host}), http.DefaultTransport), - ) + v.l = newLinktransformerMetrics(reg) + v.transportFn = func(u string) http.RoundTripper { + parsed, err := url.Parse(u) + if err != nil { + panic(err) } + return promhttp.InstrumentRoundTripperCounter( + v.l.collyRequests, + promhttp.InstrumentRoundTripperDuration(v.l.collyPerDomainLatency.MustCurryWith(prometheus.Labels{"domain": parsed.Host}), http.DefaultTransport), + ) } // Set very soft limits. @@ -299,8 +299,8 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig [] } // MustNewValidator returns mdformatter.LinkTransformer that crawls all links. -func MustNewValidator(logger log.Logger, linksValidateConfig []byte, anchorDir string, reg *prometheus.Registry) mdformatter.LinkTransformer { - v, err := NewValidator(context.TODO(), logger, linksValidateConfig, anchorDir, reg) +func MustNewValidator(logger log.Logger, linksValidateConfig []byte, anchorDir string) mdformatter.LinkTransformer { + v, err := NewValidator(context.TODO(), logger, linksValidateConfig, anchorDir, nil) if err != nil { panic(err) } @@ -360,9 +360,8 @@ func (v *validator) visit(filepath string, dest string, lineNumbers string) { v.destFutures[k] = &futureResult{cases: 1, resultFn: func() error { return nil }} matches := remoteLinkPrefixRe.FindAllStringIndex(dest, 1) if matches == nil { - if v.l != nil { - v.l.localLinksChecked.Inc() - } + v.l.localLinksChecked.Inc() + // Relative or absolute path. Check if exists. newDest := absLocalLink(v.anchorDir, filepath, dest) @@ -372,9 +371,8 @@ func (v *validator) visit(filepath string, dest string, lineNumbers string) { } return } - if v.l != nil { - v.l.remoteLinksChecked.Inc() - } + v.l.remoteLinksChecked.Inc() + validator := v.validateConfig.GetValidatorForURL(dest) if validator != nil { matched, err := validator.IsValid(k, v) diff --git a/pkg/mdformatter/linktransformer/link_test.go b/pkg/mdformatter/linktransformer/link_test.go index 37eea51..a4cb590 100644 --- a/pkg/mdformatter/linktransformer/link_test.go +++ b/pkg/mdformatter/linktransformer/link_test.go @@ -140,7 +140,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir, nil), + MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -157,7 +157,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir, nil), + MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -175,7 +175,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir, nil), + MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -197,7 +197,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir, nil), + MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -219,7 +219,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir, nil), + MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) @@ -242,7 +242,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir, nil), + MustNewValidator(logger, []byte(""), anchorDir), )) testutil.NotOk(t, err) @@ -268,7 +268,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte(""), anchorDir, nil), + MustNewValidator(logger, []byte(""), anchorDir), )) testutil.NotOk(t, err) testutil.Equals(t, fmt.Sprintf("%v%v: %v%v:1: \"https://bwplotka.dev/does-not-exists\" not accessible; status code 404: Not Found", tmpDir, filePath, relDirPath, filePath), err.Error()) @@ -283,7 +283,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka'\n type: 'ignore'\n"), anchorDir, nil), + MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka'\n type: 'ignore'\n"), anchorDir), )) testutil.Ok(t, err) }) @@ -297,7 +297,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: '(^http[s]?:\\/\\/)(www\\.)?(fakelink[0-9]\\.com\\/)'\n type: 'ignore'\n"), anchorDir, nil), + MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: '(^http[s]?:\\/\\/)(www\\.)?(fakelink[0-9]\\.com\\/)'\n type: 'ignore'\n"), anchorDir), )) testutil.Ok(t, err) }) @@ -313,7 +313,7 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Equals(t, 0, len(diff), diff.String()) _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( - MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka\\/mdox'\n token: '"+repoToken+"'\n type: 'github'\n"), anchorDir, nil), + MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka\\/mdox'\n token: '"+repoToken+"'\n type: 'github'\n"), anchorDir), )) testutil.Ok(t, err) }) diff --git a/pkg/mdformatter/linktransformer/validator.go b/pkg/mdformatter/linktransformer/validator.go index f86ab57..a9a92be 100644 --- a/pkg/mdformatter/linktransformer/validator.go +++ b/pkg/mdformatter/linktransformer/validator.go @@ -16,11 +16,10 @@ type Validator interface { // GitHubValidator.IsValid skips visiting all github issue/PR links. func (v GitHubValidator) IsValid(k futureKey, r *validator) (bool, error) { + r.l.githubSkippedLinks.Inc() + // Find rightmost index of match i.e, where regex match ends. // This will be where issue/PR number starts. Split incase of section link and convert to int. - if r.l != nil { - r.l.githubSkippedLinks.Inc() - } rightmostIndex := v._regex.FindStringIndex(k.dest) stringNum := strings.Split(k.dest[rightmostIndex[1]:], "#") num, err := strconv.Atoi(stringNum[0]) @@ -36,10 +35,8 @@ func (v GitHubValidator) IsValid(k futureKey, r *validator) (bool, error) { // RoundTripValidator.IsValid returns true if url is checked by colly. func (v RoundTripValidator) IsValid(k futureKey, r *validator) (bool, error) { + r.l.roundTripLinks.Inc() // Result will be in future. - if r.l != nil { - r.l.roundTripLinks.Inc() - } r.destFutures[k].resultFn = func() error { return r.remoteLinks[k.dest] } r.rMu.RLock() if _, ok := r.remoteLinks[k.dest]; ok { @@ -65,9 +62,8 @@ func (v RoundTripValidator) IsValid(k futureKey, r *validator) (bool, error) { // IgnoreValidator.IsValid returns true if matched so that link in not checked. func (v IgnoreValidator) IsValid(k futureKey, r *validator) (bool, error) { - if r.l != nil { - r.l.ignoreSkippedLinks.Inc() - } + r.l.ignoreSkippedLinks.Inc() + return true, nil } diff --git a/pkg/mdformatter/mdformatter.go b/pkg/mdformatter/mdformatter.go index 08c9e98..17025db 100644 --- a/pkg/mdformatter/mdformatter.go +++ b/pkg/mdformatter/mdformatter.go @@ -44,7 +44,9 @@ func newMdformatterMetrics(reg *prometheus.Registry) *mdformatterMetrics { []string{"filepath"}, ) - reg.MustRegister(m.filesProcessed, m.perFileLatency) + if reg != nil { + reg.MustRegister(m.filesProcessed, m.perFileLatency) + } return m } @@ -263,10 +265,8 @@ func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs f := New(ctx, opts...) b := bytes.Buffer{} // TODO(bwplotka): Add concurrency (collector will need to redone). - var m *mdformatterMetrics - if reg != nil { - m = newMdformatterMetrics(reg) - } + m := newMdformatterMetrics(reg) + errs := merrors.New() if spin != nil { errs.Add(spin.Start()) @@ -281,11 +281,9 @@ func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs spin.Message(fn + "...") } errs.Add(func() error { - var start_time time.Time - if reg != nil { - start_time = time.Now() - m.filesProcessed.Inc() - } + start_time := time.Now() + m.filesProcessed.Inc() + file, err := os.OpenFile(fn, os.O_RDWR, 0) if err != nil { return errors.Wrapf(err, "open %v", fn) @@ -317,10 +315,9 @@ func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs if err != nil { return errors.Wrapf(err, "write %v", fn) } - if reg != nil { - time_taken := time.Since(start_time) - m.perFileLatency.WithLabelValues(fn).Observe(time_taken.Seconds()) - } + time_taken := time.Since(start_time) + m.perFileLatency.WithLabelValues(fn).Observe(time_taken.Seconds()) + return file.Truncate(int64(n)) }()) } From eb71ddda693ed1850a8da10e85cd671f2bb8bbbe Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Mon, 2 Aug 2021 12:47:40 +0530 Subject: [PATCH 4/8] Remove server and write to file Signed-off-by: Saswata Mukherjee --- go.mod | 7 +- go.sum | 264 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- main.go | 43 +++++---- 3 files changed, 279 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index 99eb167..f338095 100644 --- a/go.mod +++ b/go.mod @@ -21,13 +21,12 @@ require ( github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 // indirect github.com/oklog/run v1.1.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.3.0 - github.com/prometheus/common v0.7.0 + github.com/prometheus/client_golang v1.11.0 + github.com/prometheus/common v0.30.0 github.com/sergi/go-diff v1.0.0 github.com/theckman/yacspin v0.8.0 github.com/yuin/goldmark v1.3.5 - golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect - golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c + golang.org/x/net v0.0.0-20210525063256-abc453219eb5 golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 diff --git a/go.sum b/go.sum index ac3b128..2ed94c2 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,43 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= contrib.go.opencensus.io/exporter/stackdriver v0.11.0/go.mod h1:hA7rlmtavV03FGxzWXAPBUnZeZBhWN/QYQAuMtxc9Bk= contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= contrib.go.opencensus.io/resource v0.0.0-20190131005048-21591786a5e0/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-amqp-common-go v1.1.3/go.mod h1:FhZtXirFANw40UXI2ntweO+VOkfaw8s6vZxUiRhLYW8= github.com/Azure/azure-amqp-common-go v1.1.4/go.mod h1:FhZtXirFANw40UXI2ntweO+VOkfaw8s6vZxUiRhLYW8= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= @@ -22,6 +53,7 @@ github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvd github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69/go.mod h1:L1AbZdiDllfyYH5l5OkAaZtk7VkWe89bPJFmnDBNHxg= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190418212003-6ac0b49e7197/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= @@ -58,6 +90,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= @@ -114,6 +147,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -152,7 +186,9 @@ github.com/efficientgo/tools/core v0.0.0-20210609125236-d73259166f20/go.mod h1:O github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20 h1:kM/ALyvAnTrwSB+nlKqoKaDnZbInp1YImZvW+gtHwc8= github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20/go.mod h1:ZV0utlglOczUWv3ih2AbqPSoLoFzdplUYxwV62eZi6Q= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanw/esbuild v0.6.5/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= @@ -169,11 +205,15 @@ github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03D github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getkin/kin-openapi v0.14.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= @@ -199,21 +239,31 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -222,20 +272,33 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040 h1:i7RUpu0EybzQyQvPT7J3MmODs4+gPcHsD/pqW0uIYVo= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99 h1:Ak8CrdlwwXwAZxzS66vgPt4U8yUZX7JwLvVR58FN5jM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.2.2/go.mod h1:7FHVg6mFpFQrjeUZrm+BaD50N5jnDKm50uVPTpyYOmU= github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= @@ -287,17 +350,23 @@ github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -362,6 +431,7 @@ github.com/muesli/smartcrop v0.3.0/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78Rwc github.com/muesli/termenv v0.8.1 h1:9q230czSP3DHVpkaPDXGp0TOfAwyjyYwXlUCQxQSaBk= github.com/muesli/termenv v0.8.1/go.mod h1:kzt/D/4a88RoheZmwfqorY3A+tnsSMA9HJC/fQSFKo0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= @@ -416,26 +486,34 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -459,6 +537,7 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -519,7 +598,10 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.31/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.3/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= @@ -537,6 +619,8 @@ go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -560,16 +644,36 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191214001246-9130b4cfad52/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -590,31 +694,52 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190420063019-afa5a82059c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c h1:KHUzaHIpjWVlVVNh65G3hhuj3KB1HnjY6Cq5cTvRQT8= golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190523182746-aaccbc9213b0/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -627,28 +752,53 @@ golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -659,18 +809,48 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca h1:pvScuB+UnCGDas2naNKUOXruM08MjwVcEdaweeynIqQ= golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -681,33 +861,83 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -716,8 +946,10 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -738,17 +970,25 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= pack.ag/amqp v0.8.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/main.go b/main.go index d9f47d8..de7eb4f 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ package main import ( "context" "fmt" - "net/http" "os" "os/signal" "path/filepath" @@ -33,7 +32,6 @@ import ( "github.com/oklog/run" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/expfmt" "gopkg.in/alecthomas/kingpin.v2" ) @@ -46,6 +44,7 @@ const ( type mdoxMetrics struct { reg *prometheus.Registry + dir string } func setupLogger(logLevel, logFormat string) log.Logger { @@ -82,7 +81,7 @@ func main() { Default(logFormatCLILog).Enum(logFormatLogfmt, logFormatJson, logFormatCLILog) // Profiling and metrics. profilesPath := app.Flag("debug.profiles", "Path to which CPU and heap profiles are saved").Hidden().String() - metrics := app.Flag("metrics", "Enable metrics and view them at https://localhost:9091/metrics").Hidden().Bool() + metrics := app.Flag("metrics", "Path to which metrics are saved in OpenMetrics format").Hidden().String() m := &mdoxMetrics{} @@ -93,6 +92,11 @@ func main() { cmd, runner := app.Parse() logger := setupLogger(*logLevel, *logFormat) + if *metrics != "" { + m.dir = *metrics + m.reg = prometheus.NewRegistry() + } + if *profilesPath != "" { finalize, err := snapshotProfiles(*profilesPath) if err != nil { @@ -110,19 +114,6 @@ func main() { cancel() }) - if *metrics { - srv := &http.Server{Addr: ":9091"} - m.reg = prometheus.NewRegistry() - - g.Add(func() error { - http.Handle("/metrics", promhttp.HandlerFor(m.reg, promhttp.HandlerOpts{})) - return srv.ListenAndServe() - }, func(err error) { - _ = srv.Shutdown(ctx) - cancel() - }) - } - // Listen for termination signals. { cancel := make(chan struct{}) @@ -178,14 +169,28 @@ func (m *mdoxMetrics) Print() error { if err != nil { return err } - - enc := expfmt.NewEncoder(os.Stdout, expfmt.FmtProtoText) + now := time.Now().UTC() + if err := os.MkdirAll(filepath.Join(m.dir, strings.ReplaceAll(now.Format(time.UnixDate), " ", "_")), os.ModePerm); err != nil { + return err + } + f, err := os.OpenFile(filepath.Join(m.dir, strings.ReplaceAll(now.Format(time.UnixDate), " ", "_"), "metrics"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) + if err != nil { + return err + } + defer f.Close() for _, mf := range mfs { - if err := enc.Encode(mf); err != nil { + for _, metric := range mf.Metric { + unixTime := now.Unix() + metric.TimestampMs = &unixTime + } + if _, err := expfmt.MetricFamilyToOpenMetrics(f, mf); err != nil { return err } } + if _, err = expfmt.FinalizeOpenMetrics(f); err != nil { + return err + } return nil } From 7d84a0efa8cf36ba06535bc96613c961a272ef14 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Wed, 4 Aug 2021 09:19:39 +0530 Subject: [PATCH 5/8] Implement suggestions Signed-off-by: Saswata Mukherjee --- main.go | 6 ++- pkg/mdformatter/linktransformer/link_test.go | 48 ++++++++++---------- pkg/mdformatter/mdformatter.go | 27 +++++++---- pkg/mdformatter/mdformatter_test.go | 4 +- pkg/transform/transform.go | 4 +- 5 files changed, 50 insertions(+), 39 deletions(-) diff --git a/main.go b/main.go index de7eb4f..dff3c37 100644 --- a/main.go +++ b/main.go @@ -266,8 +266,10 @@ This directive runs executable with arguments and put its stderr and stdout outp opts = append(opts, mdformatter.WithLinkTransformer(linktransformer.NewChain(linkTr...))) } + opts = append(opts, mdformatter.WithMetrics(m.reg)) + if *checkOnly { - diff, err := mdformatter.IsFormatted(ctx, logger, *files, m.reg, opts...) + diff, err := mdformatter.IsFormatted(ctx, logger, *files, opts...) if err != nil { return err } @@ -288,7 +290,7 @@ This directive runs executable with arguments and put its stderr and stdout outp return errors.Errorf("files not formatted: %v", diffOut) } - return mdformatter.Format(ctx, logger, *files, m.reg, opts...) + return mdformatter.Format(ctx, logger, *files, opts...) }) } diff --git a/pkg/mdformatter/linktransformer/link_test.go b/pkg/mdformatter/linktransformer/link_test.go index a4cb590..7407c8c 100644 --- a/pkg/mdformatter/linktransformer/link_test.go +++ b/pkg/mdformatter/linktransformer/link_test.go @@ -51,13 +51,13 @@ func TestLocalizer_TransformDestination(t *testing.T) { diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{ filepath.Join(tmpDir, "repo", "docs", "a", "doc.md"), filepath.Join(tmpDir, "repo", "docs", "doc2.md"), - }, nil) + }) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) }) t.Run("no domain specified", func(t *testing.T) { - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, nil, mdformatter.WithLinkTransformer( + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, mdformatter.WithLinkTransformer( NewLocalizer(logger, regexp.MustCompile(`^$`), anchorDir), )) testutil.Ok(t, err) @@ -75,7 +75,7 @@ func TestLocalizer_TransformDestination(t *testing.T) { }) t.Run("domain specified, but without full path", func(t *testing.T) { - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, nil, mdformatter.WithLinkTransformer( + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, mdformatter.WithLinkTransformer( NewLocalizer(logger, regexp.MustCompile(`myproject.example.com`), anchorDir), )) testutil.Ok(t, err) @@ -93,7 +93,7 @@ func TestLocalizer_TransformDestination(t *testing.T) { }) t.Run("domain specified", func(t *testing.T) { - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, nil, mdformatter.WithLinkTransformer( + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, mdformatter.WithLinkTransformer( NewLocalizer(logger, regexp.MustCompile(`myproject.example.com/tip/`), anchorDir), )) testutil.Ok(t, err) @@ -135,11 +135,11 @@ func TestValidator_TransformDestination(t *testing.T) { testFile := filepath.Join(tmpDir, "repo", "docs", "test", "valid-link.md") testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://bwplotka.dev/about\n"), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) @@ -152,11 +152,11 @@ func TestValidator_TransformDestination(t *testing.T) { testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://bwplotka.dev/about\n"), os.ModePerm)) testutil.Ok(t, ioutil.WriteFile(testFile2, []byte("https://bwplotka.dev/about\n"), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}, nil, mdformatter.WithLinkTransformer( + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) @@ -170,11 +170,11 @@ func TestValidator_TransformDestination(t *testing.T) { [1](.) [2](#yolo) [3](../test/valid-local-links.md) [4](../test/valid-local-links.md#yolo) [5](../a/doc.md) `), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) @@ -192,11 +192,11 @@ func TestValidator_TransformDestination(t *testing.T) { [2](#run-time-deduplication-of-ha-groups) `), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) @@ -214,11 +214,11 @@ func TestValidator_TransformDestination(t *testing.T) { [2](#hugo-का-उपयोग-करते-हुए-स्थानीय-रूप-से-साइट-चलाना) `), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte(""), anchorDir), )) testutil.Ok(t, err) @@ -237,11 +237,11 @@ func TestValidator_TransformDestination(t *testing.T) { [1](.) [2](#not-yolo) [3](../test2/invalid-local-links.md) [4](../test/invalid-local-links.md#not-yolo) [5](../test/doc.md) `), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte(""), anchorDir), )) testutil.NotOk(t, err) @@ -263,11 +263,11 @@ func TestValidator_TransformDestination(t *testing.T) { relDirPath, err := filepath.Rel(wdir, tmpDir) testutil.Ok(t, err) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte(""), anchorDir), )) testutil.NotOk(t, err) @@ -278,11 +278,11 @@ func TestValidator_TransformDestination(t *testing.T) { testFile := filepath.Join(tmpDir, "repo", "docs", "test", "invalid-link2.md") testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://www.github.com/ https://bwplotka.dev/does-not-exits\n"), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka'\n type: 'ignore'\n"), anchorDir), )) testutil.Ok(t, err) @@ -292,11 +292,11 @@ func TestValidator_TransformDestination(t *testing.T) { testFile := filepath.Join(tmpDir, "repo", "docs", "test", "links.md") testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://fakelink1.com/ http://fakelink2.com/ https://www.fakelink3.com/\n"), os.ModePerm)) - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: '(^http[s]?:\\/\\/)(www\\.)?(fakelink[0-9]\\.com\\/)'\n type: 'ignore'\n"), anchorDir), )) testutil.Ok(t, err) @@ -308,11 +308,11 @@ func TestValidator_TransformDestination(t *testing.T) { // This is substituted in config using PathorContent flag. But need to pass it directly here. repoToken := os.Getenv("GITHUB_TOKEN") - diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil) + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff), diff.String()) - _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, nil, mdformatter.WithLinkTransformer( + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n - regex: 'bwplotka\\/mdox'\n token: '"+repoToken+"'\n type: 'github'\n"), anchorDir), )) testutil.Ok(t, err) diff --git a/pkg/mdformatter/mdformatter.go b/pkg/mdformatter/mdformatter.go index 17025db..5110112 100644 --- a/pkg/mdformatter/mdformatter.go +++ b/pkg/mdformatter/mdformatter.go @@ -84,6 +84,8 @@ type Formatter struct { bm BackMatterTransformer link LinkTransformer cb CodeBlockTransformer + + reg *prometheus.Registry } // Option is a functional option type for Formatter objects. @@ -117,6 +119,13 @@ func WithCodeBlockTransformer(cb CodeBlockTransformer) Option { } } +// WithMetrics allows you to pass in Prometheus registry. +func WithMetrics(reg *prometheus.Registry) Option { + return func(m *Formatter) { + m.reg = reg + } +} + func New(ctx context.Context, opts ...Option) *Formatter { f := &Formatter{ ctx: ctx, @@ -239,33 +248,33 @@ func newSpinner(suffix string) (*yacspin.Spinner, error) { } // Format formats given markdown files in-place. IsFormatted `With...` function to see what modifiers you can add. -func Format(ctx context.Context, logger log.Logger, files []string, reg *prometheus.Registry, opts ...Option) error { +func Format(ctx context.Context, logger log.Logger, files []string, opts ...Option) error { spin, err := newSpinner(" Formatting: ") if err != nil { return err } - return format(ctx, logger, files, nil, spin, reg, opts...) + return format(ctx, logger, files, nil, spin, opts...) } // IsFormatted tries to formats given markdown files and return Diff if files are not formatted. // If diff is empty it means all files are formatted. -func IsFormatted(ctx context.Context, logger log.Logger, files []string, reg *prometheus.Registry, opts ...Option) (diffs Diffs, err error) { +func IsFormatted(ctx context.Context, logger log.Logger, files []string, opts ...Option) (diffs Diffs, err error) { d := Diffs{} spin, err := newSpinner(" Checking: ") if err != nil { return nil, err } - if err := format(ctx, logger, files, &d, spin, reg, opts...); err != nil { + if err := format(ctx, logger, files, &d, spin, opts...); err != nil { return nil, err } return d, nil } -func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs, spin *yacspin.Spinner, reg *prometheus.Registry, opts ...Option) error { +func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs, spin *yacspin.Spinner, opts ...Option) error { f := New(ctx, opts...) b := bytes.Buffer{} // TODO(bwplotka): Add concurrency (collector will need to redone). - m := newMdformatterMetrics(reg) + m := newMdformatterMetrics(f.reg) errs := merrors.New() if spin != nil { @@ -281,7 +290,7 @@ func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs spin.Message(fn + "...") } errs.Add(func() error { - start_time := time.Now() + startTime := time.Now() m.filesProcessed.Inc() file, err := os.OpenFile(fn, os.O_RDWR, 0) @@ -315,8 +324,8 @@ func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs if err != nil { return errors.Wrapf(err, "write %v", fn) } - time_taken := time.Since(start_time) - m.perFileLatency.WithLabelValues(fn).Observe(time_taken.Seconds()) + timeTaken := time.Since(startTime) + m.perFileLatency.WithLabelValues(fn).Observe(timeTaken.Seconds()) return file.Truncate(int64(n)) }()) diff --git a/pkg/mdformatter/mdformatter_test.go b/pkg/mdformatter/mdformatter_test.go index 0c5f809..1939fb6 100644 --- a/pkg/mdformatter/mdformatter_test.go +++ b/pkg/mdformatter/mdformatter_test.go @@ -42,12 +42,12 @@ func TestFormat_FormatSingle_NoTransformers(t *testing.T) { } func TestCheck_NoTransformers(t *testing.T) { - diff, err := IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/formatted.md"}, nil) + diff, err := IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/formatted.md"}) testutil.Ok(t, err) testutil.Equals(t, 0, len(diff)) testutil.Equals(t, "files the same; no diff", diff.String()) - diff, err = IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/not_formatted.md"}, nil) + diff, err = IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/not_formatted.md"}) testutil.Ok(t, err) exp, err := ioutil.ReadFile("testdata/not_formatted.md.diff") diff --git a/pkg/transform/transform.go b/pkg/transform/transform.go index 6e37c2e..aa0762d 100644 --- a/pkg/transform/transform.go +++ b/pkg/transform/transform.go @@ -101,7 +101,7 @@ func Dir(ctx context.Context, logger log.Logger, config []byte) error { } // Once we did all the changes, change links. - return mdformatter.Format(ctx, logger, tr.filesToLinkAdjust, nil, mdformatter.WithLinkTransformer(tr.linkTransformer)) + return mdformatter.Format(ctx, logger, tr.filesToLinkAdjust, mdformatter.WithLinkTransformer(tr.linkTransformer)) } type transformer struct { @@ -217,7 +217,7 @@ func (t *transformer) transformFile(path string, info os.FileInfo, err error) er return nil } - return mdformatter.Format(t.ctx, t.logger, []string{target}, nil, opts...) + return mdformatter.Format(t.ctx, t.logger, []string{target}, opts...) } type relLinkTransformer struct { From ffbc32d60e21f8f9c9508afa29a949e5c1b0fbbc Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Wed, 4 Aug 2021 09:25:30 +0530 Subject: [PATCH 6/8] Fix lint Signed-off-by: Saswata Mukherjee --- pkg/mdformatter/linktransformer/validator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mdformatter/linktransformer/validator.go b/pkg/mdformatter/linktransformer/validator.go index 7c65239..b9c824b 100644 --- a/pkg/mdformatter/linktransformer/validator.go +++ b/pkg/mdformatter/linktransformer/validator.go @@ -16,7 +16,7 @@ type Validator interface { // GitHubPullsIssuesValidator.IsValid skips visiting all github issue/PR links. func (v GitHubPullsIssuesValidator) IsValid(k futureKey, r *validator) (bool, error) { - r.l.githubSkippedLinks.Inc() + r.l.githubSkippedLinks.Inc() // Find rightmost index of match i.e, where regex match ends. // This will be where issue/PR number starts. Split incase of section link and convert to int. rightmostIndex := v._regex.FindStringIndex(k.dest) From 8f4409fb06f0f2802d776f3093bb92410986c397 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Sat, 14 Aug 2021 13:05:32 +0530 Subject: [PATCH 7/8] Implement suggestions Signed-off-by: Saswata Mukherjee --- README.md | 8 ++++++++ main.go | 48 +++++++++++++++++++++++------------------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index e96126b..64eb726 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,14 @@ Flags: --version Show application version. --log.level=info Log filtering level. --log.format=clilog Log format to use. + --profiles.path=PROFILES.PATH + Path to directory where CPU and heap profiles + will be saved; If empty, no profiling will be + enabled. + --metrics.path=METRICS.PATH + Path to directory where metrics are saved in + OpenMetrics format; If empty, no metrics will + be saved. --check If true, fmt will not modify the given files, instead it will fail if files needs formatting --code.disable-directives If false, fmt will parse custom fenced code diff --git a/main.go b/main.go index dff3c37..ba6eed2 100644 --- a/main.go +++ b/main.go @@ -42,11 +42,6 @@ const ( logFormatCLILog = "clilog" ) -type mdoxMetrics struct { - reg *prometheus.Registry - dir string -} - func setupLogger(logLevel, logFormat string) log.Logger { var lvl level.Option switch logLevel { @@ -80,23 +75,16 @@ func main() { logFormat := app.Flag("log.format", "Log format to use."). Default(logFormatCLILog).Enum(logFormatLogfmt, logFormatJson, logFormatCLILog) // Profiling and metrics. - profilesPath := app.Flag("debug.profiles", "Path to which CPU and heap profiles are saved").Hidden().String() - metrics := app.Flag("metrics", "Path to which metrics are saved in OpenMetrics format").Hidden().String() - - m := &mdoxMetrics{} + profilesPath := app.Flag("profiles.path", "Path to directory where CPU and heap profiles will be saved; If empty, no profiling will be enabled.").ExistingDir() + metricsPath := app.Flag("metrics.path", "Path to directory where metrics are saved in OpenMetrics format; If empty, no metrics will be saved.").ExistingDir() ctx, cancel := context.WithCancel(context.Background()) - registerFmt(ctx, app, m) + registerFmt(ctx, app, metricsPath) registerTransform(ctx, app) cmd, runner := app.Parse() logger := setupLogger(*logLevel, *logFormat) - if *metrics != "" { - m.dir = *metrics - m.reg = prometheus.NewRegistry() - } - if *profilesPath != "" { finalize, err := snapshotProfiles(*profilesPath) if err != nil { @@ -164,16 +152,17 @@ func snapshotProfiles(dir string) (func() error, error) { }, nil } -func (m *mdoxMetrics) Print() error { - mfs, err := m.reg.Gather() +// Dump metrics from registry into file in dir using OpenMetrics format. +func Dump(reg *prometheus.Registry, dir string) error { + mfs, err := reg.Gather() if err != nil { return err } now := time.Now().UTC() - if err := os.MkdirAll(filepath.Join(m.dir, strings.ReplaceAll(now.Format(time.UnixDate), " ", "_")), os.ModePerm); err != nil { + if err := os.MkdirAll(filepath.Join(dir, strings.ReplaceAll(now.Format(time.UnixDate), " ", "_")), os.ModePerm); err != nil { return err } - f, err := os.OpenFile(filepath.Join(m.dir, strings.ReplaceAll(now.Format(time.UnixDate), " ", "_"), "metrics"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) + f, err := os.OpenFile(filepath.Join(dir, strings.ReplaceAll(now.Format(time.UnixDate), " ", "_"), "metrics"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) if err != nil { return err } @@ -206,7 +195,7 @@ func interrupt(logger log.Logger, cancel <-chan struct{}) error { } } -func registerFmt(_ context.Context, app *extkingpin.App, m *mdoxMetrics) { +func registerFmt(_ context.Context, app *extkingpin.App, metricsPath *string) { cmd := app.Command("fmt", "Formats in-place given markdown files uniformly following GFM (Github Flavored Markdown: https://github.github.com/gfm/). Example: mdox fmt *.md") files := cmd.Arg("files", "Markdown file(s) to process.").Required().ExistingFiles() checkOnly := cmd.Flag("check", "If true, fmt will not modify the given files, instead it will fail if files needs formatting").Bool() @@ -222,8 +211,9 @@ This directive runs executable with arguments and put its stderr and stdout outp linksValidateConfig := extflag.RegisterPathOrContent(cmd, "links.validate.config", "YAML file for skipping link check, with spec defined in github.com/bwplotka/mdox/pkg/linktransformer.ValidatorConfig", extflag.WithEnvSubstitution()) cmd.Run(func(ctx context.Context, logger log.Logger) (err error) { - if m.reg != nil { - defer logerrcapture.Do(logger, m.Print, "print") + var reg *prometheus.Registry + if *metricsPath != "" { + reg = prometheus.NewRegistry() } var opts []mdformatter.Option @@ -252,7 +242,7 @@ This directive runs executable with arguments and put its stderr and stdout outp if err != nil { return err } - v, err := linktransformer.NewValidator(ctx, logger, validateConfigContent, anchorDir, m.reg) + v, err := linktransformer.NewValidator(ctx, logger, validateConfigContent, anchorDir, reg) if err != nil { return err } @@ -266,7 +256,7 @@ This directive runs executable with arguments and put its stderr and stdout outp opts = append(opts, mdformatter.WithLinkTransformer(linktransformer.NewChain(linkTr...))) } - opts = append(opts, mdformatter.WithMetrics(m.reg)) + opts = append(opts, mdformatter.WithMetrics(reg)) if *checkOnly { diff, err := mdformatter.IsFormatted(ctx, logger, *files, opts...) @@ -290,7 +280,15 @@ This directive runs executable with arguments and put its stderr and stdout outp return errors.Errorf("files not formatted: %v", diffOut) } - return mdformatter.Format(ctx, logger, *files, opts...) + if err := mdformatter.Format(ctx, logger, *files, opts...); err != nil { + return err + } + if reg != nil { + if err := Dump(reg, *metricsPath); err != nil { + return err + } + } + return nil }) } From e85a409b3332b3d6326b8a6a395528caa7ca6f9d Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Tue, 17 Aug 2021 16:42:08 +0530 Subject: [PATCH 8/8] Fix error Signed-off-by: Saswata Mukherjee --- README.md | 8 ++++++++ pkg/mdformatter/mdformatter.go | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ff433a8..099ab04 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,14 @@ Flags: --version Show application version. --log.level=info Log filtering level. --log.format=clilog Log format to use. + --profiles.path=PROFILES.PATH + Path to directory where CPU and heap profiles + will be saved; If empty, no profiling will be + enabled. + --metrics.path=METRICS.PATH + Path to directory where metrics are saved in + OpenMetrics format; If empty, no metrics will + be saved. --config-file= Path to Path to the YAML file with spec defined in github.com/bwplotka/mdox/pkg/transform.Config diff --git a/pkg/mdformatter/mdformatter.go b/pkg/mdformatter/mdformatter.go index 6ed063a..95e54bd 100644 --- a/pkg/mdformatter/mdformatter.go +++ b/pkg/mdformatter/mdformatter.go @@ -85,7 +85,7 @@ type Formatter struct { link LinkTransformer cb CodeBlockTransformer - reg *prometheus.Registry + reg *prometheus.Registry softWraps bool } @@ -120,11 +120,11 @@ func WithCodeBlockTransformer(cb CodeBlockTransformer) Option { } } - // WithMetrics allows you to pass in Prometheus registry. func WithMetrics(reg *prometheus.Registry) Option { return func(m *Formatter) { m.reg = reg + } } // WithCodeBlockTransformer allows you to override default softWrap.