# Copyright (C) 2018-2020 Arthur de Jong # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), # to deal in the Software without restriction, including without limitation # the rights to use, copy, modify, merge, publish, distribute, sublicense, # and/or sell copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. """Module that exposes the Munin data.""" import math import os import re import subprocess import time from collections import defaultdict from functools import wraps # The directory that contains the Munin data files. # This is the dbdir option in munin.conf. MUNIN_DBDIR = os.environ.get('MUNIN_DBDIR', '/var/lib/munin') def cache(function): """Cache returned data of the decorated function.""" data = dict() @wraps(function) def wrapper(*args): now = time.time() if args not in data or data[args][1] < now - 300: data[args] = (function(*args), now) return data[args][0] return wrapper def _remove_duplicates(lst): """Remove duplicate values from the list while maintaining order.""" seen = set() for x in lst: if x not in seen: seen.add(x) yield x @cache def get_info(): """Return a description of all the graphs.""" data = defaultdict(dict) # parse the datafile and group by graph with open(os.path.join(MUNIN_DBDIR, 'datafile'), 'rt', encoding='utf-8') as f: for line in f: if ':' in line: source, line = line.split(':', 1) group, host = source.split(';') key, value = line.split(' ', 1) if '.' in key: graph, key = key.split('.', 1) name = '%s/%s/%s' % (group, host, graph) data[name][key] = value.strip() # restructure graph info for name, info in data.items(): # collect field information fields = dict() for key, value in list(info.items()): if '.' in key: info.pop(key) field, key = key.split('.', 1) fields.setdefault(field, dict(name=field)) if key not in ('graph_data_size', 'update_rate'): fields[field][key] = value # remove graph=no from negative fields for _field, field_info in fields.items(): negative = field_info.get('negative') if negative: fields[negative].pop('graph', None) fields[negative].setdefault('draw', field_info.get('draw', 'LINE')) # remove graph = no and replace by removing draw for _field, field_info in fields.items(): if field_info.pop('graph', '').lower() in ('false', 'no', '0'): field_info.pop('draw', None) else: field_info.setdefault('draw', 'LINE') # expand graph_vlabel graph_vlabel = info.get('graph_vlabel', '') if '${graph_period}' in graph_vlabel: info['graph_vlabel'] = graph_vlabel.replace( '${graph_period}', info.get('graph_period', 'second')) info['name'] = name info['group'], info['host'], _ = name.split('/') graph_order = info.pop('graph_order', '') info['fields'] = [ fields[field] for field in _remove_duplicates(graph_order.split()) if field in fields] category = info.pop('graph_category', '') if category: info['category'] = category.lower() return data def _get_rrd_files(group, host, graph): """Return a list of RRD files that are available for the graph.""" files = os.listdir(os.path.join(MUNIN_DBDIR, group)) prefix = '%s-%s-' % (host, graph) return sorted( f for f in files if f.startswith(prefix) and f.endswith('.rrd')) def _fetch_rrd(filename, start, end, resolution=300, cf='AVERAGE'): """Use rrdtool to fetch values to the data.""" output = subprocess.check_output([ 'rrdtool', 'fetch', os.path.join(MUNIN_DBDIR, filename), cf, '-r', str(resolution), '-s', str(start), '-e', str(end)]) for line in output.decode('utf-8').splitlines(): if ':' in line: try: time, value = line.split(':', 1) value = float(value) if not math.isnan(value): yield int(time), value except ValueError: pass def get_raw_values(group, host, graph, start, end, resolution=300, minmax=True): """Get the data points available from the specified graph.""" start = int(start / resolution) * resolution end = int(end / resolution) * resolution data = defaultdict(defaultdict) for f in _get_rrd_files(group, host, graph): field = '-'.join(f.split('-')[2:-1]) filename = os.path.join(group, f) for time_, value in _fetch_rrd(filename, start, end, resolution, 'AVERAGE'): data[time_][field] = value if minmax: for time_, value in _fetch_rrd(filename, start, end, resolution, 'MIN'): data[time_][field + '.min'] = value for time_, value in _fetch_rrd(filename, start, end, resolution, 'MAX'): data[time_][field + '.max'] = value return [dict(time=k, **v) for k, v in sorted(data.items())] cdef_ops = { '+': (lambda a, b: a + b), '-': (lambda a, b: a - b), '*': (lambda a, b: a * b), '/': (lambda a, b: a / b), } cdef_number_re = re.compile(r'^-?[0-9]+(\.[0-9]*)?$') def cdef_eval(expression, row, suffix=''): """Evaluate a cdef expression using variables from row.""" tokens = expression.split(',') stack = [] for token in tokens: if cdef_number_re.match(token): stack.append(float(token)) elif token in cdef_ops: arg2 = stack.pop() arg1 = stack.pop() result = cdef_ops[token](arg1, arg2) stack.append(result) else: stack.append(row[token + suffix]) return stack.pop() def get_values(group, host, graph, start, end, resolution=300, minmax=True): """Get the data points available from the specified graph.""" graph_info = get_info()['%s/%s/%s' % (group, host, graph)] data = get_raw_values(group, host, graph, start, end, resolution, minmax) for field_info in graph_info['fields']: negative = field_info.get('negative') if negative: for row in data: try: values = [ -row[negative + '.min'], -row[negative], -row[negative + '.max']] ( row[negative + '.min'], row[negative], row[negative + '.max'], ) = sorted(values) except KeyError: pass cdef = field_info.get('cdef') if cdef: field = field_info['name'] for row in data: try: values = [ cdef_eval(cdef, row, '.min'), cdef_eval(cdef, row), cdef_eval(cdef, row, '.max')] row[field + '.min'], row[field], row[field + '.max'] = sorted(values) except Exception: pass return data def get_resolutions(group, host, graph): """Return a list of resolutions available for the graph.""" # find the newest file rrdfile = sorted( (os.stat(x).st_mtime, x) for x in ( os.path.join(MUNIN_DBDIR, group, y) for y in _get_rrd_files(group, host, graph)))[-1][1] output = subprocess.check_output(['rrdtool', 'info', rrdfile]) resolutions = {} rows = {} for line in output.decode('utf-8').splitlines(): if line.startswith('step = '): # the measurement resolution step = int(line.split(' = ')[1]) elif line.startswith('last_update = '): last_update = int(line.split(' = ')[1]) last_update = int(last_update / step) * step elif '.pdp_per_row = ' in line: pdp_per_row = int(line.split(' = ')[1]) resolutions[pdp_per_row * step] = line.split('.')[0] elif '.rows = ' in line: rows[line.split('.')[0]] = int(line.split(' = ')[1]) return last_update, [(r, rows[i]) for r, i in sorted(resolutions.items())]