Skip to content

Commit

Permalink
Add metrics and memory profile
Browse files Browse the repository at this point in the history 
Signed-off-by: Saswata Mukherjee <[email protected]>
  • Loading branch information
@saswatamcode
saswatamcode committed Jun 28, 2021
1 parent 4a2de89 commit e977247
Show file tree
Hide file tree
Showing 9 changed files with 223 additions and 52 deletions.
3 changes: 2 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/antchfx/xmlquery v1.3.4 // indirect
github.com/efficientgo/tools/core v0.0.0-20210609125236-d73259166f20
github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20
github.com/felixge/fgprof v0.9.1 // indirect
github.com/felixge/fgprof v0.9.1
github.com/go-kit/kit v0.10.0
github.com/gobwas/glob v0.2.3
github.com/gocolly/colly/v2 v2.1.1-0.20201013153555-8252c346cfb0
Expand All @@ -17,6 +17,7 @@ require (
github.com/mitchellh/mapstructure v1.2.2 // indirect
github.com/oklog/run v1.1.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.3.0
github.com/sergi/go-diff v1.0.0
github.com/yuin/goldmark v1.3.5
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
Expand Down
8 changes: 8 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY=
Expand All @@ -94,7 +95,9 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
Expand Down Expand Up @@ -312,6 +315,7 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw=
github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/mmark v1.3.6/go.mod h1:w7r9mkTvpS55jlfyn22qJ618itLryxXBhA7Jp3FIlkw=
Expand Down Expand Up @@ -384,21 +388,25 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc=
github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
Expand Down
52 changes: 45 additions & 7 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@ package main
import (
"context"
"fmt"
"net/http"
"os"
"os/signal"
"path/filepath"
"runtime"
"runtime/pprof"
"strings"
"syscall"

Expand All @@ -27,6 +30,8 @@ import (
"github.com/go-kit/kit/log/level"
"github.com/oklog/run"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"gopkg.in/alecthomas/kingpin.v2"
)

Expand All @@ -36,6 +41,10 @@ const (
logFormatCLILog = "clilog"
)

type mdoxMetrics struct {
reg *prometheus.Registry
}

func setupLogger(logLevel, logFormat string) log.Logger {
var lvl level.Option
switch logLevel {
Expand Down Expand Up @@ -68,10 +77,14 @@ func main() {
Default("info").Enum("error", "warn", "info", "debug")
logFormat := app.Flag("log.format", "Log format to use.").
Default(logFormatCLILog).Enum(logFormatLogfmt, logFormatJson, logFormatCLILog)
profilesPath := app.Flag("debug.profiles", "WHAT THE HECK ARE YOU DOING SO LONG").Hidden().String()
// Profiling and metrics.
profilesPath := app.Flag("debug.profiles", "Path to which CPU and heap profiles are saved").Hidden().String()
metrics := app.Flag("metrics", "Enable metrics and view them at https://localhost:9091/metrics").Hidden().Bool()

m := &mdoxMetrics{reg: nil}

ctx, cancel := context.WithCancel(context.Background())
registerFmt(ctx, app)
registerFmt(ctx, app, m)
registerTransform(ctx, app)

cmd, runner := app.Parse()
Expand All @@ -94,6 +107,19 @@ func main() {
cancel()
})

if *metrics {
srv := &http.Server{Addr: ":9091"}
m.reg = prometheus.NewRegistry()

g.Add(func() error {
http.Handle("/metrics", promhttp.HandlerFor(m.reg, promhttp.HandlerOpts{}))
return srv.ListenAndServe()
}, func(err error) {
_ = srv.Shutdown(ctx)
cancel()
})
}

// Listen for termination signals.
{
cancel := make(chan struct{})
Expand All @@ -113,25 +139,37 @@ func main() {
level.Error(logger).Log("err", errors.Wrapf(err, "%s command failed", cmd))
os.Exit(1)
}

}

func snapshotProfiles(dir string) (func() error, error) {
// TODO: now -> date
if err := os.MkdirAll(filepath.Join(dir, "now"), os.ModePerm); err != nil {
return nil, err
}
f, err := os.OpenFile(filepath.Join(dir, "now", "fgprof.pprof"), os.O_TRUNC|os.O_WRONLY, os.ModePerm)
f, err := os.OpenFile(filepath.Join(dir, "now", "fgprof.pb.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
if err != nil {
return nil, err
}

m, err := os.OpenFile(filepath.Join(dir, "now", "memprof.pb.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
if err != nil {
return nil, err
}
runtime.GC()

if err := pprof.WriteHeapProfile(m); err != nil {
return nil, err
}

fgFunc := fgprof.Start(f, fgprof.FormatPprof)

return func() (err error) {
defer errcapture.Do(&err, f.Close, "close")
return fgFunc()
}, nil
}

func interrupt(logger log.Logger, cancel <-chan struct{}) error {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
Expand All @@ -144,7 +182,7 @@ func interrupt(logger log.Logger, cancel <-chan struct{}) error {
}
}

func registerFmt(_ context.Context, app *extkingpin.App) {
func registerFmt(_ context.Context, app *extkingpin.App, m *mdoxMetrics) {
cmd := app.Command("fmt", "Formats in-place given markdown files uniformly following GFM (Github Flavored Markdown: https://github.github.com/gfm/). Example: mdox fmt *.md")
files := cmd.Arg("files", "Markdown file(s) to process.").Required().ExistingFiles()
checkOnly := cmd.Flag("check", "If true, fmt will not modify the given files, instead it will fail if files needs formatting").Bool()
Expand Down Expand Up @@ -186,7 +224,7 @@ This directive runs executable with arguments and put its stderr and stdout outp
if err != nil {
return err
}
v, err := linktransformer.NewValidator(ctx, logger, validateConfigContent, anchorDir)
v, err := linktransformer.NewValidator(ctx, logger, validateConfigContent, anchorDir, m.reg)
if err != nil {
return err
}
Expand All @@ -201,7 +239,7 @@ This directive runs executable with arguments and put its stderr and stdout outp
}

if *checkOnly {
diff, err := mdformatter.IsFormatted(ctx, logger, *files, opts...)
diff, err := mdformatter.IsFormatted(ctx, logger, *files, m.reg, opts...)
if err != nil {
return err
}
Expand All @@ -210,7 +248,7 @@ This directive runs executable with arguments and put its stderr and stdout outp
}
return errors.Errorf("files not formatted: %v", diff.String())
}
return mdformatter.Format(ctx, logger, *files, opts...)
return mdformatter.Format(ctx, logger, *files, m.reg, opts...)
})
}

Expand Down
86 changes: 83 additions & 3 deletions pkg/mdformatter/linktransformer/link.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"context"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
Expand All @@ -24,6 +25,8 @@ import (
"github.com/go-kit/kit/log/level"
"github.com/gocolly/colly/v2"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)

var remoteLinkPrefixRe = regexp.MustCompile(`^http[s]?://`)
Expand All @@ -35,6 +38,54 @@ var (
IDNotFoundErr = LookupError(errors.New("file exists, but does not have such id"))
)

type linktransformerMetrics struct {
localLinksChecked prometheus.Counter
remoteLinksChecked prometheus.Counter
roundTripLinks prometheus.Counter
githubSkippedLinks prometheus.Counter
ignoreSkippedLinks prometheus.Counter

collyRequests *prometheus.CounterVec
collyPerDomainLatency *prometheus.HistogramVec
}

func newLinktransformerMetrics(reg *prometheus.Registry) *linktransformerMetrics {
l := &linktransformerMetrics{}

l.localLinksChecked = prometheus.NewCounter(prometheus.CounterOpts{
Name: "mdox_local_links_total",
Help: "The total number of local links which were checked",
})
l.remoteLinksChecked = prometheus.NewCounter(prometheus.CounterOpts{
Name: "mdox_remote_links_total",
Help: "The total number of remote links which were checked",
})
l.roundTripLinks = prometheus.NewCounter(prometheus.CounterOpts{
Name: "mdox_round_trip_links_total",
Help: "The total number of links which were roundtrip checked",
})
l.githubSkippedLinks = prometheus.NewCounter(prometheus.CounterOpts{
Name: "mdox_github_skipped_links_total",
Help: "The total number of links which were github checked",
})
l.ignoreSkippedLinks = prometheus.NewCounter(prometheus.CounterOpts{
Name: "mdox_ignore_skipped_links_total",
Help: "The total number of links which were ignore checked",
})

l.collyRequests = prometheus.NewCounterVec(
prometheus.CounterOpts{Name: "mdox_colly_requests_total"},
[]string{},
)
l.collyPerDomainLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{Name: "mdox_colly_per_domain_latency", Buckets: prometheus.DefBuckets},
[]string{"domain"},
)

reg.MustRegister(l.localLinksChecked, l.remoteLinksChecked, l.roundTripLinks, l.githubSkippedLinks, l.ignoreSkippedLinks, l.collyRequests, l.collyPerDomainLatency)
return l
}

const (
originalURLKey = "originalURLKey"
numberOfRetriesKey = "retryKey"
Expand Down Expand Up @@ -126,6 +177,9 @@ type validator struct {

futureMu sync.Mutex
destFutures map[futureKey]*futureResult

l *linktransformerMetrics
transportFn func(url string) http.RoundTripper
}

type futureKey struct {
Expand All @@ -140,7 +194,7 @@ type futureResult struct {

// NewValidator returns mdformatter.LinkTransformer that crawls all links.
// TODO(bwplotka): Add optimization and debug modes - this is the main source of latency and pain.
func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []byte, anchorDir string) (mdformatter.LinkTransformer, error) {
func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []byte, anchorDir string, reg *prometheus.Registry) (mdformatter.LinkTransformer, error) {
var err error
config := Config{}
if string(linksValidateConfig) != "" {
Expand All @@ -149,6 +203,7 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []
return nil, err
}
}

v := &validator{
logger: logger,
anchorDir: anchorDir,
Expand All @@ -157,7 +212,26 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []
remoteLinks: map[string]error{},
c: colly.NewCollector(colly.Async(), colly.StdlibContext(ctx)),
destFutures: map[futureKey]*futureResult{},
l: nil,
transportFn: func(url string) http.RoundTripper {
return http.DefaultTransport
},
}

if reg != nil {
v.l = newLinktransformerMetrics(reg)
v.transportFn = func(u string) http.RoundTripper {
parsed, err := url.Parse(u)
if err != nil {
panic(err)
}
return promhttp.InstrumentRoundTripperCounter(
v.l.collyRequests,
promhttp.InstrumentRoundTripperDuration(v.l.collyPerDomainLatency.MustCurryWith(prometheus.Labels{"domain": parsed.Host}), http.DefaultTransport),
)
}
}

// Set very soft limits.
// E.g github has 50-5000 https://docs.github.com/en/free-pro-team@latest/rest/reference/rate-limit limit depending
// on api (only search is below 100).
Expand Down Expand Up @@ -225,8 +299,8 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []
}

// MustNewValidator returns mdformatter.LinkTransformer that crawls all links.
func MustNewValidator(logger log.Logger, linksValidateConfig []byte, anchorDir string) mdformatter.LinkTransformer {
v, err := NewValidator(context.TODO(), logger, linksValidateConfig, anchorDir)
func MustNewValidator(logger log.Logger, linksValidateConfig []byte, anchorDir string, reg *prometheus.Registry) mdformatter.LinkTransformer {
v, err := NewValidator(context.TODO(), logger, linksValidateConfig, anchorDir, reg)
if err != nil {
panic(err)
}
Expand Down Expand Up @@ -286,6 +360,9 @@ func (v *validator) visit(filepath string, dest string) {
v.destFutures[k] = &futureResult{cases: 1, resultFn: func() error { return nil }}
matches := remoteLinkPrefixRe.FindAllStringIndex(dest, 1)
if matches == nil {
if v.l != nil {
v.l.localLinksChecked.Inc()
}
// Relative or absolute path. Check if exists.
newDest := absLocalLink(v.anchorDir, filepath, dest)

Expand All @@ -295,6 +372,9 @@ func (v *validator) visit(filepath string, dest string) {
}
return
}
if v.l != nil {
v.l.remoteLinksChecked.Inc()
}
validator := v.validateConfig.GetValidatorForURL(dest)
if validator != nil {
matched, err := validator.IsValid(k, v)
Expand Down
Loading

0 comments on commit e977247

Please sign in to comment.