Skip to content

Commit

Permalink
Support modules configuration (#146)
Browse files Browse the repository at this point in the history 
* support modules configuration

Signed-off-by: Ben Ye <[email protected]>

* fallback default module if the param is missing

Signed-off-by: Ben Ye <[email protected]>

* update readme and example config file

Signed-off-by: Ben Ye <[email protected]>

* fix lint

Signed-off-by: Ben Ye <[email protected]>
  • Loading branch information
Ben Ye authored May 27, 2022
1 parent d43d3ed commit 75ae2b0
Show file tree
Hide file tree
Showing 8 changed files with 207 additions and 124 deletions.
48 changes: 25 additions & 23 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,35 +39,37 @@ $ cat examples/data.json

$ cat examples/config.yml
---
metrics:
- name: example_global_value
path: "{ .counter }"
help: Example of a top-level global value scrape in the json
labels:
environment: beta # static label
location: "planet-{.location}" # dynamic label

- name: example_value
type: object
help: Example of sub-level value scrapes from a json
path: '{.values[?(@.state == "ACTIVE")]}'
labels:
environment: beta # static label
id: '{.id}' # dynamic label
values:
active: 1 # static value
count: '{.count}' # dynamic value
boolean: '{.some_boolean}'

headers:
X-Dummy: my-test-header
modules:
default:
metrics:
- name: example_global_value
path: "{ .counter }"
help: Example of a top-level global value scrape in the json
labels:
environment: beta # static label
location: "planet-{.location}" # dynamic label

- name: example_value
type: object
help: Example of sub-level value scrapes from a json
path: '{.values[?(@.state == "ACTIVE")]}'
labels:
environment: beta # static label
id: '{.id}' # dynamic label
values:
active: 1 # static value
count: '{.count}' # dynamic value
boolean: '{.some_boolean}'

headers:
X-Dummy: my-test-header

$ python -m SimpleHTTPServer 8000 &
Serving HTTP on 0.0.0.0 port 8000 ...

$ ./json_exporter --config.file examples/config.yml &

$ curl "http://localhost:7979/probe?target=http://localhost:8000/examples/data.json" | grep ^example
$ curl "http://localhost:7979/probe?module=default&target=http://localhost:8000/examples/data.json" | grep ^example
example_global_value{environment="beta",location="planet-mars"} 1234
example_value_active{environment="beta",id="id-A"} 1
example_value_active{environment="beta",id="id-C"} 1
Expand Down
15 changes: 13 additions & 2 deletions cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package cmd
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"

Expand Down Expand Up @@ -86,9 +87,19 @@ func probeHandler(w http.ResponseWriter, r *http.Request, logger log.Logger, con
defer cancel()
r = r.WithContext(ctx)

module := r.URL.Query().Get("module")
if module == "" {
module = "default"
}
if _, ok := config.Modules[module]; !ok {
http.Error(w, fmt.Sprintf("Unknown module %q", module), http.StatusBadRequest)
level.Debug(logger).Log("msg", "Unknown module", "module", module)
return
}

registry := prometheus.NewPedanticRegistry()

metrics, err := exporter.CreateMetricsList(config)
metrics, err := exporter.CreateMetricsList(config.Modules[module])
if err != nil {
level.Error(logger).Log("msg", "Failed to create metrics list from config", "err", err)
}
Expand All @@ -102,7 +113,7 @@ func probeHandler(w http.ResponseWriter, r *http.Request, logger log.Logger, con
return
}

fetcher := exporter.NewJSONFetcher(ctx, logger, config, r.URL.Query())
fetcher := exporter.NewJSONFetcher(ctx, logger, config.Modules[module], r.URL.Query())
data, err := fetcher.FetchJSON(target)
if err != nil {
http.Error(w, "Failed to fetch JSON response. TARGET: "+target+", ERROR: "+err.Error(), http.StatusServiceUnavailable)
Expand Down
107 changes: 83 additions & 24 deletions cmd/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ func TestFailIfSelfSignedCA(t *testing.T) {
}))
defer target.Close()

req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
recorder := httptest.NewRecorder()
probeHandler(recorder, req, log.NewNopLogger(), config.Config{})
probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"default": {}}})

resp := recorder.Result()
body, _ := ioutil.ReadAll(resp.Body)
Expand All @@ -45,13 +45,21 @@ func TestFailIfSelfSignedCA(t *testing.T) {
}

func TestSucceedIfSelfSignedCA(t *testing.T) {
c := config.Config{}
c.HTTPClientConfig.TLSConfig.InsecureSkipVerify = true
c := config.Config{
Modules: map[string]config.Module{
"default": {
HTTPClientConfig: pconfig.HTTPClientConfig{
TLSConfig: pconfig.TLSConfig{
InsecureSkipVerify: true,
},
},
}},
}
target := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
}))
defer target.Close()

req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
recorder := httptest.NewRecorder()
probeHandler(recorder, req, log.NewNopLogger(), c)

Expand All @@ -63,6 +71,29 @@ func TestSucceedIfSelfSignedCA(t *testing.T) {
}
}

func TestDefaultModule(t *testing.T) {
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
}))
defer target.Close()

req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
recorder := httptest.NewRecorder()
probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"default": {}}})

resp := recorder.Result()
if resp.StatusCode != http.StatusOK {
t.Fatalf("Default module test fails unexpectedly, expected 200, got %d", resp.StatusCode)
}

// Module doesn't exist.
recorder = httptest.NewRecorder()
probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"foo": {}}})
resp = recorder.Result()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("Default module test fails unexpectedly, expected 400, got %d", resp.StatusCode)
}
}

func TestFailIfTargetMissing(t *testing.T) {
req := httptest.NewRequest("GET", "http://example.com/foo", nil)
recorder := httptest.NewRecorder()
Expand All @@ -86,9 +117,9 @@ func TestDefaultAcceptHeader(t *testing.T) {
}))
defer target.Close()

req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
recorder := httptest.NewRecorder()
probeHandler(recorder, req, log.NewNopLogger(), config.Config{})
probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"default": {}}})

resp := recorder.Result()
body, _ := ioutil.ReadAll(resp.Body)
Expand Down Expand Up @@ -118,7 +149,7 @@ func TestCorrectResponse(t *testing.T) {
t.Fatalf("Failed to load config file %s", test.ConfigFile)
}

req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL+test.ServeFile, nil)
req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL+test.ServeFile, nil)
recorder := httptest.NewRecorder()
probeHandler(recorder, req, log.NewNopLogger(), c)

Expand All @@ -145,15 +176,21 @@ func TestBasicAuth(t *testing.T) {
}))
defer target.Close()

req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
recorder := httptest.NewRecorder()
c := config.Config{}
auth := &pconfig.BasicAuth{
Username: username,
Password: pconfig.Secret(password),
c := config.Config{
Modules: map[string]config.Module{
"default": {
HTTPClientConfig: pconfig.HTTPClientConfig{
BasicAuth: &pconfig.BasicAuth{
Username: username,
Password: pconfig.Secret(password),
},
},
},
},
}

c.HTTPClientConfig.BasicAuth = auth
probeHandler(recorder, req, log.NewNopLogger(), c)

resp := recorder.Result()
Expand All @@ -175,11 +212,16 @@ func TestBearerToken(t *testing.T) {
}))
defer target.Close()

req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
recorder := httptest.NewRecorder()
c := config.Config{}
c := config.Config{
Modules: map[string]config.Module{"default": {
HTTPClientConfig: pconfig.HTTPClientConfig{
BearerToken: pconfig.Secret(token),
},
}},
}

c.HTTPClientConfig.BearerToken = pconfig.Secret(token)
probeHandler(recorder, req, log.NewNopLogger(), c)

resp := recorder.Result()
Expand All @@ -206,10 +248,15 @@ func TestHTTPHeaders(t *testing.T) {
}))
defer target.Close()

req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
recorder := httptest.NewRecorder()
c := config.Config{}
c.Headers = headers
c := config.Config{
Modules: map[string]config.Module{
"default": {
Headers: headers,
},
},
}

probeHandler(recorder, req, log.NewNopLogger(), c)

Expand Down Expand Up @@ -264,9 +311,15 @@ func TestBodyPostTemplate(t *testing.T) {
w.WriteHeader(http.StatusOK)
}))

req := httptest.NewRequest("POST", "http://example.com/foo"+"?target="+target.URL, strings.NewReader(test.Body.Content))
req := httptest.NewRequest("POST", "http://example.com/foo"+"?module=default&target="+target.URL, strings.NewReader(test.Body.Content))
recorder := httptest.NewRecorder()
c := config.Config{Body: test.Body}
c := config.Config{
Modules: map[string]config.Module{
"default": {
Body: test.Body,
},
},
}

probeHandler(recorder, req, log.NewNopLogger(), c)

Expand Down Expand Up @@ -351,15 +404,21 @@ func TestBodyPostQuery(t *testing.T) {
w.WriteHeader(http.StatusOK)
}))

req := httptest.NewRequest("POST", "http://example.com/foo"+"?target="+target.URL, strings.NewReader(test.Body.Content))
req := httptest.NewRequest("POST", "http://example.com/foo"+"?module=default&target="+target.URL, strings.NewReader(test.Body.Content))
q := req.URL.Query()
for k, v := range test.QueryParams {
q.Add(k, v)
}
req.URL.RawQuery = q.Encode()

recorder := httptest.NewRecorder()
c := config.Config{Body: test.Body}
c := config.Config{
Modules: map[string]config.Module{
"default": {
Body: test.Body,
},
},
}

probeHandler(recorder, req, log.NewNopLogger(), c)

Expand Down
27 changes: 17 additions & 10 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,13 @@ const (
ValueTypeUntyped ValueType = "untyped"
)

// Config contains metrics and headers defining a configuration
// Config contains multiple modules.
type Config struct {
Modules map[string]Module `yaml:"modules"`
}

// Module contains metrics and headers defining a configuration
type Module struct {
Headers map[string]string `yaml:"headers,omitempty"`
Metrics []Metric `yaml:"metrics"`
HTTPClientConfig pconfig.HTTPClientConfig `yaml:"http_client_config,omitempty"`
Expand All @@ -71,15 +76,17 @@ func LoadConfig(configPath string) (Config, error) {
}

// Complete Defaults
for i := 0; i < len(config.Metrics); i++ {
if config.Metrics[i].Type == "" {
config.Metrics[i].Type = ValueScrape
}
if config.Metrics[i].Help == "" {
config.Metrics[i].Help = config.Metrics[i].Name
}
if config.Metrics[i].ValueType == "" {
config.Metrics[i].ValueType = ValueTypeUntyped
for _, module := range config.Modules {
for i := 0; i < len(module.Metrics); i++ {
if module.Metrics[i].Type == "" {
module.Metrics[i].Type = ValueScrape
}
if module.Metrics[i].Help == "" {
module.Metrics[i].Help = module.Metrics[i].Name
}
if module.Metrics[i].ValueType == "" {
module.Metrics[i].ValueType = ValueTypeUntyped
}
}
}

Expand Down
Loading

0 comments on commit 75ae2b0

Please sign in to comment.