Improve build and execution

Dockerfile

@@ -1,19 +1,18 @@
-FROM golang:alpine
+FROM alpine:3.4
 
-#RUN apt-get update &&\
-#    rm -rf /var/lib/apt/lists
-RUN apk --no-cache --update add git ca-certificates
-WORKDIR $GOPATH/src/app/
-ADD . . 
-RUN go get -v
-RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o nginx-vts-exporter .
-RUN mv $GOPATH/src/app/nginx-vts-exporter /
-COPY ./docker-entrypoint.sh /
+# Creating directory for our application
+RUN mkdir /app
 
-EXPOSE 9113
+# Copying entrypoint script
+COPY ./docker-entrypoint.sh /app/
+# Copying VTS Metrics exporter binary
+COPY bin/nginx-vts-exporter /app/
 
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["/nginx-vts-exporter"]
-#, "-nginx.scrape_uri=http://localhost/status/format/json"]
+ENV NGIX_HOST http://localhost
+ENV METRICS_ENDPOINT "/metrics"
+ENV METRICS_ADDR ":9913"
+ENV DEFAULT_METRICS_NS "nginx"
 
-#CMD ["$GOPATH/src/app/nginx-vts-exporter"] 
+EXPOSE 9913
+
+ENTRYPOINT ["/app/docker-entrypoint.sh"]

Dockerfile-build

@@ -0,0 +1,9 @@
+FROM golang:alpine
+
+RUN apk --no-cache --update add git ca-certificates
+WORKDIR $GOPATH/src/app/
+ADD . .
+RUN go get -v
+RUN mkdir /build
+RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o nginx-vts-exporter .
+RUN mv $GOPATH/src/app/nginx-vts-exporter /build/

README.md

@@ -1,40 +1,57 @@
-nginx-vts-exporter
-===
+#nginx-vts-exporter
+
 Simple server that scrapes Nginx vts stats and exports them via HTTP for Prometheus consumption
 
-Dependency
----
+#Dependency
+
 * [nginx-module-vts](https://github.com/vozlt/nginx-module-vts)
 * [Prometheus](https://prometheus.io/)
 * [Golang](https://golang.org/)
 
-Compile
----
+#Download
+Binary can be downloaded from `bin` directory.
+Latest version v0.0.3
+
 ```
-go get -v ./...
-go build
+# SHA512 Sum
+16eec84a6496529ef76a83af54f659111abecca6bcb4b2edd0b327223f93e735ae4aca2078bf4c41fded831c3d116170b277d194af64074f45992191e3a7bfb6  bin/nginx-vts-exporter
 ```
 
-Run
----
+#Compile
+
 ```
-nohup ./nginx-vts-exporter -nginx.scrape_uri=http://localhost/status/format/json
+$ ./build-binary.sh
 ```
+This shell script above will build a temp Docker image with the binary and then
+export the binary inside ./bin/ directory
 
-Dockerize
---
+#Run
 
-Build
 ```
-docker build -t vts-export .
+$ nohup /bin/nginx-vts-exporter -nginx.scrape_uri=http://localhost/status/format/json
 ```
-Run
+
+#Dockerized
+To Dockerize this application yo need to pass two steps the build then the containerization.
+
+## Environment variables
+This image is configurable using different env variables
+
+Variable name | Default     | Description
+------------- | ----------- | --------------
+NGINX_STATUS |  http://localhost/status/format/json | Nginx JSON format status page
+METRICS_ENDPOINT | /metrics  | Metrics endpoint exportation URI
+METRICS_ADDR | :9913 | Metrics exportation address:port
+METRICS_NS | nginx | Prometheus metrics Namespaces
+
+
+##Build
 ```
-docker run -ti vts-export
+$ ./build-binary.sh
+$ docker build -t vts-export .
 ```
 
-Run with args
+##Run
 ```
-docker run -ti vts-export -nginx.scrape_uri=http://localhost/status/format/json
+docker run  -ti --rm --env NGIX_HOST="http://localhost/status/format/json" --env METRICS_NS="nginx_prod1" vts-export
 ```
-

build-binary.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+random=`awk -v min=10000 -v max=99999 'BEGIN{srand(); print int(min+rand()*(max-min+1))}'`
+tag_name="build_$random"
+bin_diretory=`pwd`/bin
+
+echo "[Step 1] - Building binary inside image"
+docker build --tag=$tag_name \
+             --file=Dockerfile-build . 
+
+echo "[Step 2] - Copying the binary from docker image"
+docker run --rm --volume=$bin_diretory:/output $tag_name cp /build/nginx-vts-exporter /output/nginx-vts-exporter >/dev/null 2>&1
+
+echo "[Step 3] - Tranfering ownership to current user"
+sudo chown -R `whoami`:`whoami` $bin_diretory >/dev/null 2>&1
+
+echo "[Step 4] - Cleaning the tmp build images"
+docker rmi -f $tag_name >/dev/null 2>&1
+
+echo "[Success] - Build complete !!"

docker-entrypoint.sh

@@ -1,10 +1,19 @@
-#!/bin/sh 
+#!/bin/sh
 set -eo pipefail
+binary="/app/nginx-vts-exporter"
+default_status="$NGIX_HOST/status/format/json"
+NGINX_STATUS=${NGINX_STATUS:-$default_status}
+METRICS_NS=${METRICS_NS:-$DEFAULT_METRICS_NS}
 
 # If there are any arguments then we want to run those instead
-if [[ "$1" == "/nginx-vts-exporter" || -z $1 ]]; then
-  exec "$@"
-#$GOPATH/src/app/nginx-vts-exporter
-else
-  exec "/nginx-vts-exporter" "$@"
-fi
+#if [[ "$1" == "$binary" || -z $1 ]]; then
+#  exec "$@"
+#else
+#  echo "Running the default"
+#echo "[$0] - Nginx scrape host --> [$NGINX_STATUS]"
+#echo "[$0] - Metrics Address   --> [$METRICS_ADDR]"
+#echo "[$0] - Metrics Endpoint  --> [$METRICS_ENDPOINT]"
+#echo "[$0] - Metrics Namespace  --> [$METRICS_NS]"
+#echo "[$0] - Running metrics nginx-vts-exporter"
+exec "$binary" -nginx.scrape_uri=$NGINX_STATUS -telemetry.address $METRICS_ADDR -telemetry.endpoint $METRICS_ENDPOINT -metrics.namespace $METRICS_NS
+#fi

metrics.md

@@ -0,0 +1,55 @@
+# Metrics documentation
+This file contain documentation about exposed Prometheus metrics
+
+#Server main
+## Metrics details
+Nginx data         | Name                            | Exposed informations     
+------------------ | ------------------------------- | ------------------------
+ **Connections**   | `{NAMESPACE}_server_connections`| status [active, reading, writing, waiting, accepted, handled]
+
+##Metrics output example
+```
+# Server Connections
+nginx_server_connections{status="accepted"} 70606
+```
+
+#Server zones
+## Metrics details
+Nginx data         | Name                            | Exposed informations     
+------------------ | ------------------------------- | ------------------------
+ **Requests**      | `{NAMESPACE}server_requests`    | code [2xx, 3xx, 4xx, 5xx, total], host _(or domain name)_
+ **Bytes**         | `{NAMESPACE}server_bytes`       | direction [in, out], host _(or domain name)_
+ **Cache**         | `{NAMESPACE}server_cache`       | status [bypass, expired, hit, miss, revalidated, scarce, stale, updating], host _(or domain name)_
+
+##Metrics output example
+```
+# Server Requests
+nginx_server_requests{code="1xx",host="test.domain.com"} 0
+
+# Server Bytes
+nginx_server_bytes{direction="in",host="test.domain.com"} 21
+
+# Server Cache
+nginx_server_cache{host="test.domain.com",status="bypass"} 2
+```
+
+#Upstreams
+## Metrics details
+Nginx data         | Name                            | Exposed informations     
+------------------ | ------------------------------- | ------------------------
+ **Requests**      | `{NAMESPACE}_upstream_requests` | code [2xx, 3xx, 4xx, 5xx and total], upstream _(or upstream name)_
+ **Bytes**         | `{NAMESPACE}_upstream_bytes`    | direction [in, out], upstream _(or upstream name)_
+ **Response time** | `{NAMESPACE}_upstream_response` | backend (or server), in_bytes, out_bytes, upstream _(or upstream name)_
+ **Requests/sec**  | `NOT EXPORTED YET`              |
+
+##Metrics output example
+```
+# Upstream Requests
+nginx_upstream_requests{code="1xx",upstream="XXX-XXXXX-3000"} 0
+
+# Upstream Bytes
+nginx_upstream_bytes{direction="in",upstream="XXX-XXXXX-3000"} 0
+
+# Upstream Response time
+nginx_upstream_response{backend="10.2.15.10:3000",upstream="XXX-XXXXX-3000"} 99
+```

nginx_vts_exporter.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"sync"
 	"time"
+  "strconv"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -135,6 +136,11 @@ type Cache struct {
 	} `json:"overCounts"`
 }
 
+func FloatToString(input_num float64) string {
+    // to convert a float number to a string
+    return strconv.FormatFloat(input_num, 'f', 6, 64)
+}
+
 const namespace = "nginx"
 
 type Exporter struct {
@@ -144,7 +150,7 @@ type Exporter struct {
 	serverMetrics, upstreamMetrics, cacheMetrics map[string]*prometheus.GaugeVec
 }
 
-func newServerMetric(metricName string, docString string, labels []string) *prometheus.GaugeVec {
+func newServerMetric(metricName string, docString string, labels []string, namespace string) *prometheus.GaugeVec {
 	return prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: namespace,
@@ -155,7 +161,7 @@ func newServerMetric(metricName string, docString string, labels []string) *prom
 	)
 }
 
-func newUpstreamMetric(metricName string, docString string, labels []string) *prometheus.GaugeVec {
+func newUpstreamMetric(metricName string, docString string, labels []string, namespace string) *prometheus.GaugeVec {
 	return prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: namespace,
@@ -166,7 +172,7 @@ func newUpstreamMetric(metricName string, docString string, labels []string) *pr
 	)
 }
 
-func newCacheMetric(metricName string, docString string, labels []string) *prometheus.GaugeVec {
+func newCacheMetric(metricName string, docString string, labels []string, namespace string) *prometheus.GaugeVec {
 	return prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: namespace,
@@ -177,23 +183,25 @@ func newCacheMetric(metricName string, docString string, labels []string) *prome
 	)
 }
 
-func NewExporter(uri string) *Exporter {
+func NewExporter(uri string, namespace string) *Exporter {
 
 	return &Exporter{
 		URI: uri,
 		serverMetrics: map[string]*prometheus.GaugeVec{
-			"connections": newServerMetric("connections", "nginx connections", []string{"status"}),
-			"requests":    newServerMetric("requests", "requests counter", []string{"host", "code"}),
-			"bytes":       newServerMetric("bytes", "request/response bytes", []string{"host", "direction"}),
-			"cache":       newServerMetric("cache", "cache counter", []string{"host", "status"}),
+			"connections": newServerMetric("connections", "nginx connections", []string{"status"}, namespace),
+			"requests":    newServerMetric("requests", "requests counter", []string{"host", "code"}, namespace),
+			"bytes":       newServerMetric("bytes", "request/response bytes", []string{"host", "direction"}, namespace),
+			"cache":       newServerMetric("cache", "cache counter", []string{"host", "status"}, namespace),
 		},
 		upstreamMetrics: map[string]*prometheus.GaugeVec{
-			"requests": newUpstreamMetric("requests", "requests counter", []string{"upstream", "code"}),
-			"bytes":    newUpstreamMetric("bytes", "request/response bytes", []string{"upstream", "direction"}),
+			"requests": newUpstreamMetric("requests", "requests counter", []string{"upstream", "code"}, namespace),
+			"bytes":    newUpstreamMetric("bytes", "request/response bytes", []string{"upstream", "direction"}, namespace),
+			"response":    newUpstreamMetric("response", "request response time", []string{"upstream", "backend"}, namespace),
+
 		},
 		cacheMetrics: map[string]*prometheus.GaugeVec{
-			"requests": newCacheMetric("requests", "cache requests counter", []string{"zone", "status"}),
-			"bytes":    newCacheMetric("bytes", "cache request/response bytes", []string{"zone", "direction"}),
+			"requests": newCacheMetric("requests", "cache requests counter", []string{"zone", "status"}, namespace),
+			"bytes":    newCacheMetric("bytes", "cache request/response bytes", []string{"zone", "direction"}, namespace),
 		},
 	}
 }
@@ -305,6 +313,8 @@ func (e *Exporter) scrape() {
 
 			e.upstreamMetrics["bytes"].WithLabelValues(name, "in").Add(float64(s.InBytes))
 			e.upstreamMetrics["bytes"].WithLabelValues(name, "out").Add(float64(s.OutBytes))
+
+			e.upstreamMetrics["response"].WithLabelValues(name, s.Server).Add(float64(s.ResponseMsec))
 		}
 	}
 
@@ -340,16 +350,17 @@ func fetchHTTP(uri string, timeout time.Duration) func() (io.ReadCloser, error)
 }
 
 var (
-	listenAddress   = flag.String("telemetry.address", ":9113", "Address on which to expose metrics.")
+	listenAddress   = flag.String("telemetry.address", ":9913", "Address on which to expose metrics.")
 	metricsEndpoint = flag.String("telemetry.endpoint", "/metrics", "Path under which to expose metrics.")
+	metricsNamespace = flag.String("metrics.namespace", "nginx", "Prometheus metrics namespace.")
 	nginxScrapeURI  = flag.String("nginx.scrape_uri", "http://localhost/status", "URI to nginx stub status page")
 	insecure        = flag.Bool("insecure", true, "Ignore server certificate if using https")
 )
 
 func main() {
 	flag.Parse()
 
-	exporter := NewExporter(*nginxScrapeURI)
+	exporter := NewExporter(*nginxScrapeURI, *metricsNamespace)
 	prometheus.MustRegister(exporter)
 	prometheus.Unregister(prometheus.NewProcessCollector(os.Getpid(), ""))
 	prometheus.Unregister(prometheus.NewGoCollector())
@@ -365,6 +376,9 @@ func main() {
 			</html>`))
 	})
 
-	log.Printf("Starting Server: %s", *listenAddress)
+	log.Printf("Starting Server at : %s", *listenAddress)
+	log.Printf("Metrics endpoint: %s", *metricsEndpoint)
+	log.Printf("Metrics namespace: %s", *metricsNamespace)
+	log.Printf("Scraping information from : %s", *nginxScrapeURI)
 	log.Fatal(http.ListenAndServe(*listenAddress, nil))
 }