Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add sum, livemin, and livemax multiprocess modes for Gauges #794

Merged
merged 3 commits into from
Apr 25, 2022
Merged

Add sum, livemin, and livemax multiprocess modes for Gauges #794

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
59347f0
add sum, livemin, and livemax multiprocessing modes
JoshKarpel Apr 7, 2022
2b820c4
apply suggestions from @nkov
JoshKarpel Apr 8, 2022
157cb03
call it a label instead of a tag
JoshKarpel Apr 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -614,9 +614,12 @@ Gauges have several modes they can run in, which can be selected with the `multi

- 'all': Default. Return a timeseries per process alive or dead.
- 'liveall': Return a timeseries per process that is still alive.
- 'sum': Return a single timeseries that is the sum of the values of all processes, alive or dead.
- 'livesum': Return a single timeseries that is the sum of the values of alive processes.
- 'max': Return a single timeseries that is the maximum of the values of all processes, alive or dead.
- 'livemax': Return a single timeseries that is the maximum of the values of alive processes.
- 'min': Return a single timeseries that is the minimum of the values of all processes, alive or dead.
- 'livemin': Return a single timeseries that is the minimum of the values of alive processes.
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

since the support is comprehensive with this change, wonder if we should simplify this to be something like:

- 'all': Default. Return a timeseries per process (alive or dead)
- 'sum': Return a single timeseries that is the some of the values of all processes (alive or dead).
- 'max': Return a single timeseries that is the maximum of the values of all processes (alive or dead).
- 'min': Return a single timeseries that is the minimum of the values of all processes (alive or dead).

Prepend 'live' to the beginning of the mode to return the same result but only considering living processes
- eg: 'liveall, 'livesum', 'livemax', 'livemin'


```python
from prometheus_client import Gauge
Expand Down
2 changes: 1 addition & 1 deletion prometheus_client/metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ def f():
d.set_function(lambda: len(my_dict))
"""
_type = 'gauge'
_MULTIPROC_MODES = frozenset(('min', 'max', 'livesum', 'liveall', 'all'))
_MULTIPROC_MODES = frozenset(('min', 'livemin', 'max', 'livemax', 'sum', 'livesum', 'all', 'liveall'))

def __init__(self,
name: str,
Expand Down
16 changes: 8 additions & 8 deletions prometheus_client/multiprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import os
import warnings

from .metrics import Gauge
from .metrics_core import Metric
from .mmap_dict import MmapedDict
from .samples import Sample
Expand Down Expand Up @@ -63,7 +64,7 @@ def _parse_key(key):
try:
file_values = MmapedDict.read_all_values_from_file(f)
except FileNotFoundError:
if typ == 'gauge' and parts[1] in ('liveall', 'livesum'):
if typ == 'gauge' and 'live' in parts[1]:
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe parts[1].startswith('live')?

# Those files can disappear between the glob of collect
# and now (via a mark_process_dead call) so don't fail if
# the file is missing
Expand Down Expand Up @@ -96,15 +97,15 @@ def _accumulate_metrics(metrics, accumulate):
name, labels, value, timestamp, exemplar = s
if metric.type == 'gauge':
without_pid_key = (name, tuple(l for l in labels if l[0] != 'pid'))
if metric._multiprocess_mode == 'min':
if metric._multiprocess_mode in ('min', 'livemin'):
current = samples_setdefault(without_pid_key, value)
if value < current:
samples[without_pid_key] = value
elif metric._multiprocess_mode == 'max':
elif metric._multiprocess_mode in ('max', 'livemax'):
current = samples_setdefault(without_pid_key, value)
if value > current:
samples[without_pid_key] = value
elif metric._multiprocess_mode == 'livesum':
elif metric._multiprocess_mode in ('sum', 'livesum'):
samples[without_pid_key] += value
else: # all/liveall
samples[(name, labels)] = value
Expand Down Expand Up @@ -156,7 +157,6 @@ def mark_process_dead(pid, path=None):
"""Do bookkeeping for when one process dies in a multi-process setup."""
if path is None:
path = os.environ.get('PROMETHEUS_MULTIPROC_DIR', os.environ.get('prometheus_multiproc_dir'))
for f in glob.glob(os.path.join(path, f'gauge_livesum_{pid}.db')):
os.remove(f)
for f in glob.glob(os.path.join(path, f'gauge_liveall_{pid}.db')):
os.remove(f)
for mode in {m for m in Gauge._MULTIPROC_MODES if 'live' in m}:
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if we agree on the previous one then probably would want to make the same change here: m.startswith('live')

for f in glob.glob(os.path.join(path, f'gauge_{mode}_{pid}.db')):
os.remove(f)
33 changes: 33 additions & 0 deletions tests/test_multiprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,17 @@ def test_gauge_min(self):
g2.set(2)
self.assertEqual(1, self.registry.get_sample_value('g'))

def test_gauge_livemin(self):
g1 = Gauge('g', 'help', registry=None, multiprocess_mode='livemin')
values.ValueClass = MultiProcessValue(lambda: 456)
g2 = Gauge('g', 'help', registry=None, multiprocess_mode='livemin')
self.assertEqual(0, self.registry.get_sample_value('g'))
g1.set(1)
g2.set(2)
self.assertEqual(1, self.registry.get_sample_value('g'))
mark_process_dead(123, os.environ['PROMETHEUS_MULTIPROC_DIR'])
self.assertEqual(2, self.registry.get_sample_value('g'))

def test_gauge_max(self):
g1 = Gauge('g', 'help', registry=None, multiprocess_mode='max')
values.ValueClass = MultiProcessValue(lambda: 456)
Expand All @@ -141,6 +152,28 @@ def test_gauge_max(self):
g2.set(2)
self.assertEqual(2, self.registry.get_sample_value('g'))

def test_gauge_livemax(self):
g1 = Gauge('g', 'help', registry=None, multiprocess_mode='livemax')
values.ValueClass = MultiProcessValue(lambda: 456)
g2 = Gauge('g', 'help', registry=None, multiprocess_mode='livemax')
self.assertEqual(0, self.registry.get_sample_value('g'))
g1.set(2)
g2.set(1)
self.assertEqual(2, self.registry.get_sample_value('g'))
mark_process_dead(123, os.environ['PROMETHEUS_MULTIPROC_DIR'])
self.assertEqual(1, self.registry.get_sample_value('g'))

def test_gauge_sum(self):
g1 = Gauge('g', 'help', registry=None, multiprocess_mode='sum')
values.ValueClass = MultiProcessValue(lambda: 456)
g2 = Gauge('g', 'help', registry=None, multiprocess_mode='sum')
self.assertEqual(0, self.registry.get_sample_value('g'))
g1.set(1)
g2.set(2)
self.assertEqual(3, self.registry.get_sample_value('g'))
mark_process_dead(123, os.environ['PROMETHEUS_MULTIPROC_DIR'])
self.assertEqual(3, self.registry.get_sample_value('g'))

def test_gauge_livesum(self):
g1 = Gauge('g', 'help', registry=None, multiprocess_mode='livesum')
values.ValueClass = MultiProcessValue(lambda: 456)
Expand Down