Merge remote-tracking branch 'upstream/master'

README.md

@@ -26,6 +26,7 @@ Note that in order to use the image, it is necessary to accept the terms of the
 - **LANG** - Set this to the language you would like the license to be printed in.
 - **MQ_QMGR_NAME** - Set this to the name you want your Queue Manager to be created with.
 - **LOG_FORMAT** - Set this to change the format of the logs which are printed on the container's stdout.  Set to "json" to use JSON format (JSON object per line); set to "basic" to use a simple human-readable format.  Defaults to "basic".
+- **MQ_ENABLE_METRICS** - Set this to `true` to generate Prometheus metrics for your Queue Manager.
 
 See the [default developer configuration docs](docs/developer-config.md) for the extra environment variables supported by the MQ Advanced for Developers image.
 

cmd/runmqserver/mqconfig_linux.go

@@ -33,6 +33,72 @@ var fsTypes = map[int64]string{
 	0x9123683e: "btrfs",
 	0x01021994: "tmpfs",
 	0x794c7630: "overlayfs",
+	0x58465342: "xfs",
+	// less popular codes
+	0xadf5:     "adfs",
+	0xadff:     "affs",
+	0x5346414F: "afs",
+	0x0187:     "autofs",
+	0x73757245: "coda",
+	0x28cd3d45: "cramfs",
+	0x453dcd28: "cramfs",
+	0x64626720: "debugfs",
+	0x73636673: "securityfs",
+	0xf97cff8c: "selinux",
+	0x43415d53: "smack",
+	0x858458f6: "ramfs",
+	0x958458f6: "hugetlbfs",
+	0x73717368: "squashfs",
+	0xf15f:     "ecryptfs",
+	0x414A53:   "efs",
+	0xabba1974: "xenfs",
+	0x3434:     "nilfs",
+	0xF2F52010: "f2fs",
+	0xf995e849: "hpfs",
+	0x9660:     "isofs",
+	0x72b6:     "jffs2",
+	0x6165676C: "pstorefs",
+	0xde5e81e4: "efivarfs",
+	0x00c0ffee: "hostfs",
+	0x137F:     "minix_14",  // minix v1 fs, 14 char names
+	0x138F:     "minix_30",  // minix v1 fs, 30 char names
+	0x2468:     "minix2_14", // minix v2 fs, 14 char names
+	0x2478:     "minix2_30", // minix v2 fs, 30 char names
+	0x4d5a:     "minix3_60", // minix v3 fs, 60 char names
+	0x4d44:     "msdos",
+	0x564c:     "ncp",
+	0x7461636f: "ocfs2",
+	0x9fa1:     "openprom",
+	0x002f:     "qnx4",
+	0x68191122: "qnx6",
+	0x6B414653: "afs_fs",
+	0x52654973: "reiserfs",
+	0x517B:     "smb",
+	0x27e0eb:   "cgroup",
+	0x63677270: "cgroup2",
+	0x7655821:  "rdtgroup",
+	0x57AC6E9D: "stack_end",
+	0x74726163: "tracefs",
+	0x01021997: "v9fs",
+	0x62646576: "bdevfs",
+	0x64646178: "daxfs",
+	0x42494e4d: "binfmtfs",
+	0x1cd1:     "devpts",
+	0xBAD1DEA:  "futexfs",
+	0x50495045: "pipefs",
+	0x9fa0:     "proc",
+	0x534F434B: "sockfs",
+	0x62656572: "sysfs",
+	0x9fa2:     "usbdevice",
+	0x11307854: "mtd_inode",
+	0x09041934: "anon_inode",
+	0x73727279: "btrfs",
+	0x6e736673: "nsfs",
+	0xcafe4a11: "bpf",
+	0x5a3c69f0: "aafs",
+	0x15013346: "udf",
+	0x13661366: "balloon_kvm",
+	0x58295829: "zsmalloc",
 }
 
 func checkFS(path string) error {
@@ -43,7 +109,11 @@ func checkFS(path string) error {
 		return nil
 	}
 	// Use a type conversion to make type an int64.  On s390x it's a uint32.
-	t := fsTypes[int64(statfs.Type)]
+	t, ok := fsTypes[int64(statfs.Type)]
+	if !ok {
+		log.Printf("WARNING: detected %v has unknown filesystem type %x", path, statfs.Type)
+		return nil
+	}
 	switch t {
 	case "aufs", "overlayfs", "tmpfs":
 		return fmt.Errorf("%v uses unsupported filesystem type: %v", path, t)

docs/internals.md

@@ -24,6 +24,7 @@ The `runmqserver` command has the following responsibilities:
     - Works as PID 1, so is responsible for [reaping zombie processes](https://blog.phusion.nl/2015/01/20/docker-and-the-pid-1-zombie-reaping-problem/)
 * Creating and starting a queue manager
 * Configuring the queue manager, by running any MQSC scripts found under `/etc/mqm`
+* Starting Prometheus metrics generation for the queue manager (if enabled)
 * Indicates to the `chkmqready` command that configuration is complete, and that normal readiness checking can happen.  This is done by writing a file into `/run/runmqserver`
 
 In addition, for MQ Advanced for Developers only, the web server is started.
@@ -35,4 +36,17 @@ The `runmqdevserver` command is added to the MQ Advanced for Developers image on
 2. Generates MQSC files to put in `/etc/mqm`, based on a template, which is updated with values based on supplied environment variables.
 3. If requested, it creates TLS key stores under `/run/runmqdevserver`, and configures MQ and the web server to use them
 
-A special version of `runmqserver` is used in the developer image, which performs extra actions like starting the web server.  This is built using the `mqdev` [build constraint](https://golang.org/pkg/go/build/#hdr-Build_Constraints).
+A special version of `runmqserver` is used in the developer image, which performs extra actions like starting the web server.  This is built using the `mqdev` [build constraint](https://golang.org/pkg/go/build/#hdr-Build_Constraints).
+
+## Prometheus metrics
+[Prometheus](https://prometheus.io) metrics are generated for the queue manager as follows:
+
+1. A connection is established with the queue manager
+2. Metrics are discovered by subscribing to topics that provide meta-data on metric classes, types and elements
+3. Subscriptions are then created for each topic that provides this metric data
+4. Metrics are initialised using Prometheus names mapped from their element descriptions
+5. The metrics are then registered with the Prometheus registry as Prometheus Gauges
+6. Publications are processed on a periodic basis to retrieve the metric data
+7. An HTTP server is setup to listen for requests from Prometheus on `/metrics` port `9157`
+8. Prometheus requests are handled by updating the Prometheus Gauges with the latest metric data
+9. These updated Prometheus Gauges are then collected by the Prometheus registry

docs/usage.md

@@ -37,6 +37,21 @@ docker run \
 
 The Docker image always uses `/mnt/mqm` for MQ data, which is correctly linked for you under `/var/mqm` at runtime.  This is to handle problems with file permissions on some platforms.
 
+## Running with the default configuration and Prometheus metrics enabled
+You can run a queue manager with [Prometheus](https://prometheus.io) metrics enabled.  The following command will generate Prometheus metrics for your queue manager on `/metrics` port `9157`:
+
+```
+docker run \
+  --env LICENSE=accept \
+  --env MQ_QMGR_NAME=QM1 \
+  --env MQ_ENABLE_METRICS=true \
+  --publish 1414:1414 \
+  --publish 9443:9443 \
+  --publish 9157:9157 \
+  --detach \
+  ibmcom/mq
+```
+
 ## Customizing the queue manager configuration
 
 You can customize the configuration in several ways:

install-mq.sh

@@ -139,7 +139,7 @@ rm -rf ${DIR_EXTRACT}
 
 # Apply any bug fixes not included in base Ubuntu or MQ image.
 # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run
-$UBUNTU && apt-get upgrade -y gpgv gnupg
+$UBUNTU && apt-get install -y gnupg gpgv libgcrypt20 perl-base --only-upgrade
 # End of bug fixes
 
 # Clean up cached files

internal/metrics/exporter.go

@@ -33,6 +33,7 @@ const (
 type exporter struct {
 	qmName       string
 	gaugeMap     map[string]*prometheus.GaugeVec
+	counterMap   map[string]*prometheus.CounterVec
 	firstCollect bool
 	log          *logger.Logger
 }
@@ -41,6 +42,7 @@ func newExporter(qmName string, log *logger.Logger) *exporter {
 	return &exporter{
 		qmName:       qmName,
 		gaugeMap:     make(map[string]*prometheus.GaugeVec),
+		counterMap:   make(map[string]*prometheus.CounterVec),
 		firstCollect: true,
 		log:          log,
 	}
@@ -54,12 +56,22 @@ func (e *exporter) Describe(ch chan<- *prometheus.Desc) {
 
 	for key, metric := range response {
 
-		// Allocate a Prometheus Gauge for each available metric
-		gaugeVec := createGaugeVec(metric.name, metric.description, metric.objectType)
-		e.gaugeMap[key] = gaugeVec
+		if metric.isDelta {
+			// For delta type metrics - allocate a Prometheus Counter
+			counterVec := createCounterVec(metric.name, metric.description, metric.objectType)
+			e.counterMap[key] = counterVec
 
-		// Describe metric
-		gaugeVec.Describe(ch)
+			// Describe metric
+			counterVec.Describe(ch)
+
+		} else {
+			// For non-delta type metrics - allocate a Prometheus Gauge
+			gaugeVec := createGaugeVec(metric.name, metric.description, metric.objectType)
+			e.gaugeMap[key] = gaugeVec
+
+			// Describe metric
+			gaugeVec.Describe(ch)
+		}
 	}
 }
 
@@ -71,49 +83,88 @@ func (e *exporter) Collect(ch chan<- prometheus.Metric) {
 
 	for key, metric := range response {
 
-		// Reset Prometheus Gauge
-		gaugeVec := e.gaugeMap[key]
-		gaugeVec.Reset()
-
-		// Populate Prometheus Gauge with metric values
-		// - Skip on first collect to avoid build-up of accumulated values
-		if !e.firstCollect {
-			for label, value := range metric.values {
-				var err error
-				var gauge prometheus.Gauge
-
-				if label == qmgrLabelValue {
-					gauge, err = gaugeVec.GetMetricWithLabelValues(e.qmName)
-				} else {
-					gauge, err = gaugeVec.GetMetricWithLabelValues(label, e.qmName)
+		if metric.isDelta {
+			// For delta type metrics - update their Prometheus Counter
+			counterVec := e.counterMap[key]
+
+			// Populate Prometheus Counter with metric values
+			// - Skip on first collect to avoid build-up of accumulated values
+			if !e.firstCollect {
+				for label, value := range metric.values {
+					var err error
+					var counter prometheus.Counter
+
+					if label == qmgrLabelValue {
+						counter, err = counterVec.GetMetricWithLabelValues(e.qmName)
+					} else {
+						counter, err = counterVec.GetMetricWithLabelValues(label, e.qmName)
+					}
+					if err == nil {
+						counter.Add(value)
+					} else {
+						e.log.Errorf("Metrics Error: %s", err.Error())
+					}
 				}
-				if err == nil {
-					gauge.Set(value)
-				} else {
-					e.log.Errorf("Metrics Error: %s", err.Error())
+			}
+
+			// Collect metric
+			counterVec.Collect(ch)
+
+		} else {
+			// For non-delta type metrics - reset their Prometheus Gauge
+			gaugeVec := e.gaugeMap[key]
+			gaugeVec.Reset()
+
+			// Populate Prometheus Gauge with metric values
+			// - Skip on first collect to avoid build-up of accumulated values
+			if !e.firstCollect {
+				for label, value := range metric.values {
+					var err error
+					var gauge prometheus.Gauge
+
+					if label == qmgrLabelValue {
+						gauge, err = gaugeVec.GetMetricWithLabelValues(e.qmName)
+					} else {
+						gauge, err = gaugeVec.GetMetricWithLabelValues(label, e.qmName)
+					}
+					if err == nil {
+						gauge.Set(value)
+					} else {
+						e.log.Errorf("Metrics Error: %s", err.Error())
+					}
 				}
 			}
-		}
 
-		// Collect metric
-		gaugeVec.Collect(ch)
+			// Collect metric
+			gaugeVec.Collect(ch)
+		}
 	}
 
 	if e.firstCollect {
 		e.firstCollect = false
 	}
 }
 
+// createCounterVec returns a Prometheus CounterVec populated with metric details
+func createCounterVec(name, description string, objectType bool) *prometheus.CounterVec {
+
+	prefix, labels := getVecDetails(objectType)
+
+	counterVec := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      prefix + "_" + name,
+			Help:      description,
+		},
+		labels,
+	)
+	return counterVec
+}
+
 // createGaugeVec returns a Prometheus GaugeVec populated with metric details
 func createGaugeVec(name, description string, objectType bool) *prometheus.GaugeVec {
 
-	prefix := qmgrPrefix
-	labels := []string{qmgrLabel}
-
-	if objectType {
-		prefix = objectPrefix
-		labels = []string{objectLabel, qmgrLabel}
-	}
+	prefix, labels := getVecDetails(objectType)
 
 	gaugeVec := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
@@ -125,3 +176,16 @@ func createGaugeVec(name, description string, objectType bool) *prometheus.Gauge
 	)
 	return gaugeVec
 }
+
+// getVecDetails returns the required prefix and labels for a metric
+func getVecDetails(objectType bool) (prefix string, labels []string) {
+
+	prefix = qmgrPrefix
+	labels = []string{qmgrLabel}
+
+	if objectType {
+		prefix = objectPrefix
+		labels = []string{objectLabel, qmgrLabel}
+	}
+	return prefix, labels
+}