Skip to content

Commit

Permalink
mvcc: Export a "Last DB compaction" timestamp metric (#12176)
Browse files Browse the repository at this point in the history
This is to aid with debugging the effectiveness of systems that
manually take care of cluster compaction, and have greater visibity
into recent compactions.

It can be handy to alert on the exactly how long it was since a
compaction (and also to put on dashboards) had happened.

---

Tested using a test cluster, the final result looks like this:

```
	root@etcd-1:~# ETCDCTL_API=3 /tmp/test-etcd/etcdctl --endpoints=192.168.232.10:2379 compact 1012
	compacted revision 1012

	root@etcd-1:~# curl -s 192.168.232.10:2379/metrics | grep last
	# HELP etcd_debugging_mvcc_db_compaction_last The unix time since the last db compaction.  Resets to 0 on start.
	# TYPE etcd_debugging_mvcc_db_compaction_last gauge
	etcd_debugging_mvcc_db_compaction_last 1.595873939e+09
```
  • Loading branch information
benjojo authored Aug 26, 2020
1 parent facd0c9 commit c20cc05
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 0 deletions.
1 change: 1 addition & 0 deletions mvcc/kvstore_compaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ func (s *store) scheduleCompaction(compactMainRev int64, keep map[revision]struc
defer func() { dbCompactionTotalMs.Observe(float64(time.Since(totalStart) / time.Millisecond)) }()
keyCompactions := 0
defer func() { dbCompactionKeysCounter.Add(float64(keyCompactions)) }()
defer func() { dbCompactionLast.Set(float64(time.Now().Unix())) }()

end := make([]byte, 8)
binary.BigEndian.PutUint64(end, uint64(compactMainRev+1))
Expand Down
9 changes: 9 additions & 0 deletions mvcc/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,14 @@ var (
Buckets: prometheus.ExponentialBuckets(100, 2, 14),
})

dbCompactionLast = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "etcd_debugging",
Subsystem: "mvcc",
Name: "db_compaction_last",
Help: "The unix time of the last db compaction. Resets to 0 on start.",
})

dbCompactionKeysCounter = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "etcd_debugging",
Expand Down Expand Up @@ -324,6 +332,7 @@ func init() {
prometheus.MustRegister(indexCompactionPauseMs)
prometheus.MustRegister(dbCompactionPauseMs)
prometheus.MustRegister(dbCompactionTotalMs)
prometheus.MustRegister(dbCompactionLast)
prometheus.MustRegister(dbCompactionKeysCounter)
prometheus.MustRegister(dbTotalSize)
prometheus.MustRegister(dbTotalSizeDebug)
Expand Down

0 comments on commit c20cc05

Please sign in to comment.