# HG changeset patch # User Tero Marttila # Date 1295911686 -7200 # Node ID 47e977c23ba2acddd5b8ac75a90167ae71de3115 # Parent cd9ca8068b093434d475566e750d824cba1c486b implement rendering of pmacct rrd graphs, and a dir/top.png feature diff -r cd9ca8068b09 -r 47e977c23ba2 rrdweb/backend.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rrdweb/backend.py Tue Jan 25 01:28:06 2011 +0200 @@ -0,0 +1,92 @@ +from rrdweb import rrd + +import operator + +def host_def (idx, name, dir, ds_in, ds_out, cf) : + """ + VDEFs for given host, giving its avg/max values: + + DEF:in/out_{idx} + CDEFtraf_{idx} + VDEF:avg/max_{idx} + PRINT:{idx} avg/max {value} + """ + + params = dict( + dir=dir, ds_in=ds_in, ds_out=ds_out, cf=cf, + idx=idx, rrd='%s/%s.rrd' % (dir, name) + ) + + return [ + # in/out bandwidth in bytes/s + 'DEF:in_%(idx)d=%(rrd)s:%(ds_in)s:%(cf)s' % params, + 'DEF:out_%(idx)d=%(rrd)s:%(ds_out)s:%(cf)s' % params, + + # total traffic in bits/s + 'CDEF:traf_%(idx)d=in_%(idx)d,out_%(idx)d,+,8,*' % params, + + # average + maximum values + 'VDEF:avg_%(idx)d=traf_%(idx)d,AVERAGE' % params, + 'VDEF:max_%(idx)d=traf_%(idx)d,MAXIMUM' % params, + + # output + 'PRINT:avg_%(idx)d:%(idx)d avg %%lf' % params, + 'PRINT:max_%(idx)d:%(idx)d max %%lf' % params, + ] + +def parse_report (rrds, lines) : + """ + Parse the report output into a [ (name, avg, max) ] list + """ + + # idx values by type + data = { + 'avg': {}, + 'max': {}, + } + + # interpret output + for line in lines : + # parse + idx, type, value = line.split() + + # convert + idx = int(idx) + value = float(value) + + # store + data[type][idx] = value + + # build into name : (values) dict + return [ + ( + (name, data['avg'][idx], data['max'][idx]) + ) for idx, name in enumerate(rrds) + ] + +def calc_top_hosts (dir, rrds, ds_in, ds_out, period='15m', count=5, cf='AVERAGE') : + """ + Return the list of top-N rrd's in the given dir, sorted by total in/out bandwidth average over the given period. + """ + + # vdefs for hosts, avg/max_ in bits/s + prints + defs = [stmt for idx, name in enumerate(rrds) for stmt in host_def(idx, name, dir, ds_in, ds_out, cf)] + + # execute + _, _, output, _ = rrd.graph(False, *defs, start=('-%s' % period)) + + # parse + data = parse_report(rrds, output) + + # sort for top-N avg + data.sort(key=operator.itemgetter(1)) + top_avg = data[:count] + + # sort for top-N max + data.sort(key=operator.itemgetter(2)) + top_max = data[:count] + + # merge hosts (we lose sorting order) + return set(host for host, avg, max in top_avg + top_max) + + diff -r cd9ca8068b09 -r 47e977c23ba2 rrdweb/graph.py --- a/rrdweb/graph.py Tue Jan 25 01:19:40 2011 +0200 +++ b/rrdweb/graph.py Tue Jan 25 01:28:06 2011 +0200 @@ -42,21 +42,21 @@ def overview_opts () : """ - Common options for the overview graph + Graph statements for a single-source overview graph """ return dict( width = 600, height = 50, ), [ - "CDEF:all=in,out,+", + "CDEF:all=in0,out0,+", "VDEF:max=all,MAXIMUM", "VDEF:avg=all,AVERAGE", "VDEF:min=all,MINIMUM", - "LINE1:in#0000FF:In", - "LINE1:out#00CC00:Out", + "LINE1:in0#0000FF:In", + "LINE1:out0#00CC00:Out", "GPRINT:max:%6.2lf %Sbps max", "GPRINT:avg:%6.2lf %Sbps avg", @@ -65,7 +65,7 @@ def detail_opts () : """ - Common options for the detail graph + Common options for a single-source detail graph """ return dict( @@ -74,14 +74,14 @@ height = 200, ), [ # values - 'VDEF:in_max=in,MAXIMUM', - 'VDEF:in_avg=in,AVERAGE', - 'VDEF:in_min=in,MINIMUM', - 'VDEF:in_cur=in,LAST', - 'VDEF:out_max=out,MAXIMUM', - 'VDEF:out_avg=out,AVERAGE', - 'VDEF:out_min=out,MINIMUM', - 'VDEF:out_cur=out,LAST', + 'VDEF:in_max=in0,MAXIMUM', + 'VDEF:in_avg=in0,AVERAGE', + 'VDEF:in_min=in0,MINIMUM', + 'VDEF:in_cur=in0,LAST', + 'VDEF:out_max=out0,MAXIMUM', + 'VDEF:out_avg=out0,AVERAGE', + 'VDEF:out_min=out0,MINIMUM', + 'VDEF:out_cur=out0,LAST', # legend/graph "COMMENT:%4s" % "", @@ -90,13 +90,13 @@ "COMMENT:%11s" % "Minimum", "COMMENT:%11s\\l" % "Current", - "LINE1:in#0000FF:%4s" % "In", + "LINE1:in0#0000FF:%4s" % "In", 'GPRINT:in_max:%6.2lf %Sbps', 'GPRINT:in_avg:%6.2lf %Sbps', 'GPRINT:in_min:%6.2lf %Sbps', 'GPRINT:in_cur:%6.2lf %Sbps\\l', - "LINE1:out#00CC00:%4s" % "Out", + "LINE1:out0#00CC00:%4s" % "Out", 'GPRINT:out_max:%6.2lf %Sbps', 'GPRINT:out_avg:%6.2lf %Sbps', 'GPRINT:out_min:%6.2lf %Sbps', @@ -106,11 +106,84 @@ "COMMENT:Generated %s\\r" % timestamp().replace(':', '\\:'), ] +# set of line colors used for a multi-source graph, in rgb hex form (no prefix) +MULTI_COLORS = ( + '00cc00', + '0000ff', + 'cc0000', +) + +def multi_graph_source_defs (idx, name, color) : + """ + Render graph and legend summary for a single source in a multi-source graph. + """ + + params = dict( + idx = idx, name = name, color = color + ) + + return [stmt.format(**params) for stmt in ( + "CDEF:all{idx}=in{idx},out{idx},+", + + "VDEF:max{idx}=all{idx},MAXIMUM", + "VDEF:avg{idx}=all{idx},AVERAGE", + "VDEF:cur{idx}=all{idx},LAST", + + "LINE1:all{idx}#{color}: ", + + "GPRINT:max{idx}:%6.2lf %Sbps", + "GPRINT:avg{idx}:%6.2lf %Sbps", + "GPRINT:cur{idx}:%6.2lf %Sbps", + "COMMENT:\t{name}\\l", + )] + + +def multi_graph (sources) : + """ + Graph definition for a multi-source overview graph. + + Uses combined in/out totals, giving a single line per source. + """ + + # defs for each source + sources_defs = [multi_graph_source_defs(idx, name, color) for (idx, name), color in zip(sources, MULTI_COLORS)] + + return dict( + # dimensions + width = 600, + height = 200, + + ), [ + # legend header + "COMMENT: ", + "COMMENT:%11s" % "Maximum", + "COMMENT:%11s" % "Average", + "COMMENT:%11s" % "Current", + "COMMENT:\t%s\\l" % "", + + ] + [ + # each source + stmt for defs in sources_defs for stmt in defs + ] + STYLE_DEFS = { 'overview': overview_opts, 'detail': detail_opts, } +def hourly_opts (title) : + + return dict( + # labels + x_grid = None, + + # general info + title = "Hourly %s" % (title, ), + + # interval + start = "-1h", + ) + def daily_opts (title) : """ Common options for the 'daily' view @@ -153,52 +226,99 @@ INTERVAL_DEFS = { + 'hourly': hourly_opts, 'daily': daily_opts, 'weekly': weekly_opts, 'yearly': yearly_opts, } -def mrtg_data (rrd_path) : +def data_defs (idx, rrd, ds_in, ds_out, bytes=True, cf='AVERAGE') : """ - Data sources for network in/out from an MRTG rrd + Generate the DEF/CDEF statements for the in{idx}/out{idx} data sources for the given RRD and DS names. + + If bytes is given, convert the value into bits. """ - return [ + params = dict( + ds_in=ds_in, ds_out=ds_out, cf=cf, + idx=idx, rrd=rrd, + ) + + if bytes : # data sources, bytes/s - r'DEF:in0=%s:ds0:AVERAGE' % rrd_path, - r'DEF:out0=%s:ds1:AVERAGE' % rrd_path, - + yield 'DEF:in_raw{idx}={rrd}:{ds_in}:{cf}'.format(**params) + yield 'DEF:out_raw{idx}={rrd}:{ds_out}:{cf}'.format(**params) + # data, bits/s - 'CDEF:in=in0,8,*', - 'CDEF:out=out0,8,*', + yield 'CDEF:in{idx}=in_raw{idx},8,*'.format(**params) + yield 'CDEF:out{idx}=out_raw{idx},8,*'.format(**params) + + else : + # data sources, bits/s + yield 'DEF:in{idx}={rrd}:{ds_in}:{cf}'.format(**params) + yield 'DEF:out{idx}={rrd}:{ds_out}:{cf}'.format(**params) - ] -def collectd_data (rrd_path) : +def mrtg_data (idx, rrd) : + """ + Generate the in{idx}/out{idx} data sources fro the given MRTG rrd. + """ + + return data_defs(idx, rrd, 'ds0', 'ds1') + +def collectd_data (idx, rrd) : """ Data sources for if_octets from a collectd rrd """ - - return [ - # data sources, bytes/s - r'DEF:in0=%s:rx:AVERAGE' % rrd_path, - r'DEF:out0=%s:tx:AVERAGE' % rrd_path, + + return data_defs(idx, rrd, 'rx', 'tx') - # data, bits/s - 'CDEF:in=in0,8,*', - 'CDEF:out=out0,8,*', +def pmacct_data (idx, rrd) : + """ + Data sources for in/out bytes from a pmacct rrd + """ + + return data_defs(idx, rrd, 'in', 'out') - ] - -def _graph (style, interval, title, data_func, rrd_path, out_path) : +def graph_single (style, interval, title, data_func, rrd_path, out_path) : + """ + Render graph. + + Returns a (width, height, print_lines, graph_file) tuple. + """ + style_opts, style_vars = STYLE_DEFS[style]() interval_opts = INTERVAL_DEFS[interval](title) opts = rrd.merge_opts(common_opts(), style_opts, interval_opts) - data = data_func(rrd_path) + style_vars + data = list(data_func(0, rrd_path)) + style_vars return rrd.graph(out_path, *data, **opts) +def graph_multi (interval, title, rrd_list, data_func, out) : + """ + Render a multi-source graph. + + interval - the name of the time interval to use + title - graph title + rrd_list - sequence of (rrd, name) tuples for each source to draw + data_func - the data source definition to use + out - output path for graph + """ + + # data sources + data_defs = [stmt for idx, (path, name) in enumerate(rrd_list) for stmt in data_func(idx, path)] + + # options + graph_opts, graph_defs = multi_graph([(idx, name) for idx, (path, name) in enumerate(rrd_list)]) + interval_opts = INTERVAL_DEFS[interval](title) + + # combine + opts = rrd.merge_opts(common_opts(), graph_opts, interval_opts) + defs = data_defs + graph_defs + + # graph + return rrd.graph(out, *defs, **opts) def mrtg (style, interval, title, rrd_path, out_path) : return _graph(style, interval, title, mrtg_data, rrd_path, out_path) @@ -206,3 +326,7 @@ def collectd_ifoctets (style, interval, title, rrd_path, out_path) : return _graph(style, interval, title, collectd_data, rrd_path, out_path) +def pmacct_bytes (style, interval, title, rrd_path, out_path) : + return _graph(style, interval, title, pmacct_data, rrd_path, out_path) + + diff -r cd9ca8068b09 -r 47e977c23ba2 rrdweb/rrd.py --- a/rrdweb/rrd.py Tue Jan 25 01:19:40 2011 +0200 +++ b/rrdweb/rrd.py Tue Jan 25 01:28:06 2011 +0200 @@ -4,7 +4,9 @@ import rrdtool +import tempfile import logging +import os log = logging.getLogger('rrdweb.rrd') @@ -89,16 +91,58 @@ return func(*args) -def graph (out_path, *args, **opts) : +def graph (out, *args, **opts) : """ - Create a graph from data stored in one or several RRDs. + Render a graph image and/or print a report from data stored in one or several RRDs. - Graph image output is written to the given path. + If the output path is given, the graph data is written to the given file, otherwise a temporary file is + created. The resulting output file is returned as an opened file object, ready to read() the graph data. - Returns... something to do with the image's dimensions, or even the data itself? + In the temporary file case, the returned file is unlinked before being returned. + + If out is explicitly given as False, no graph output is generated, and graph_file is returned as None. + + Returns a (width, height, [ report output lines ], graph_file) tuple. """ + + # open/tempfile + if out is None : + # tempfile + tmp_fd, out_path = tempfile.mkstemp('.png') - return run_cmd(rrdtool.graph, (out_path, ), opts, args) + # disregard the tmp_fd, we re-open the file after it's been written out + os.close(tmp_fd) + + elif out is False : + # no output + tmp_fd = None + out_path = '' + + else : + # direct output + tmp_fd = None + out_path = out + + # render + try : + # invoke + width, height, output = run_cmd(rrdtool.graph, (out_path, ), opts, args) + + # return resulting file + if out_path : + out_file = open(out_path, 'r') + + else : + # no graph output + out_file = None + + return width, height, output, out_file + + finally : + if tmp_fd : + # cleanup tempfile + # XXX: definately not portable to windows + os.unlink(out_path) def create (rrd_path, *args, **opts) : """ diff -r cd9ca8068b09 -r 47e977c23ba2 rrdweb/wsgi.py --- a/rrdweb/wsgi.py Tue Jan 25 01:19:40 2011 +0200 +++ b/rrdweb/wsgi.py Tue Jan 25 01:28:06 2011 +0200 @@ -7,7 +7,7 @@ from werkzeug import Request, Response from werkzeug.routing import Map, Rule -from rrdweb import html, graph +from rrdweb import html, graph, backend import os, os.path import errno @@ -19,20 +19,22 @@ class WSGIApp (object) : - def __init__ (self, rrdpath, tplpath, imgpath) : + def __init__ (self, rrdpath, tplpath, imgpath, rrdgraph=graph.pmacct_bytes) : """ Configure rrdpath - path to directory containing *.rrd files tplpath - path to HTML templates imgpath - path to generated PNG images. Must be writeable + + rrdgraph - the graph.*_data function for rendering the graphs. """ self.rrdpath = os.path.abspath(rrdpath) self.templates = html.BaseFormatter(tplpath) + self.imgpath = os.path.abspath(imgpath) - # XXX: some kind of fancy cache thingie :) - self.imgpath = os.path.abspath(imgpath) + self.rrd_graph_func = rrdgraph # wrap to use werkzeug's Request/Response @@ -79,6 +81,25 @@ return response + def fs_path (self, name) : + """ + Lookup and return the full filesystem path to the given relative RRD file/dir. + + The given name must be a relative path (no leading /). + + Raises NotFound for invalid paths. + """ + + # full path + path = os.path.normpath(os.path.join(self.rrdpath, name)) + + # check inside base path + if not path.startswith(self.rrdpath) : + # not found + raise exceptions.NotFound(name) + + # ok + return path def scan_dir (self, dir) : """ @@ -253,29 +274,13 @@ return self.templates.render('target', title = self.rrd_title(rrd), + hourly_img = url(self.graph, rrd=rrd, style='detail', interval='hourly'), daily_img = url(self.graph, rrd=rrd, style='detail', interval='daily'), weekly_img = url(self.graph, rrd=rrd, style='detail', interval='weekly'), yearly_img = url(self.graph, rrd=rrd, style='detail', interval='yearly'), ) - def fs_path (self, node) : - """ - Lookup and return the full filesystem path to the given relative RRD/dir path. - """ - - # dir is relative (no leading slash) - # full path - path = os.path.normpath(os.path.join(self.rrdpath, node)) - - # check inside base path - if not path.startswith(self.rrdpath) : - # mask - raise exceptions.NotFound(node) - - # ok - return path - def rrd_path (self, rrd) : """ Lookup and return the full filesystem path to the given RRD name. @@ -300,7 +305,7 @@ def render_graph (self, rrd, style, interval, png_path) : """ - Render the given graph for the given RRD to the given path. + Render the given graph for the given RRD to the given path, returning the opened file object. """ rrd_path = self.rrd_path(rrd) @@ -311,8 +316,11 @@ log.debug("%s -> %s", rrd_path, png_path) - # XXX: always generate - graph.mrtg(style, interval, title, rrd_path, png_path) + # generate using the variant function given + w, h, report, png_file = graph.graph_single(style, interval, title, self.rrd_graph_func, rrd_path, png_path) + + # the open'd .tmp file + return png_file def rrd_graph (self, rrd, style, interval, flush=False) : """ @@ -356,41 +364,32 @@ # re-generate to tmp file tmp_path = os.path.join(self.imgpath, style, interval, rrd) + '.tmp' - - self.render_graph(rrd, style, interval, tmp_path) + + # open and write the graph image + img_file = self.render_graph(rrd, style, interval, tmp_path) # replace .png with .tmp (semi-atomic, but atomic enough..) + # XXX: probably not portable to windows, what with img_file os.rename(tmp_path, img_path) - - # open the now-fresh .png and return that - return open(img_path) - - + + else : + # use existing file + img_file = open(img_path, 'rb') + + # return the now-fresh .png and return that + return img_file ### Request handlers - - -# def node (self, url, path) : -# """ -# Arbitrate between URLs to dirs and to RRDs. -# """ -# - - def index (self, req, url, dir = '') : """ Directory overview dir - (optional) relative path to subdir from base rrdpath """ - - # lookup fs path + + # lookup path = self.fs_path(dir) - # found? - if not os.path.isdir(path) : - raise exceptions.NotFound("No such RRD directory: %s" % (dir, )) - # scan subdirs, rrds = self.scan_dir(path) @@ -444,14 +443,42 @@ # respond with file wrapper return Response(response_file, mimetype='image/png', direct_passthrough=True) + + def graph_top (self, req, url, dir = '', count=5) : + """ + Show top N hosts by peak/average. + """ + # find + path = self.fs_path(dir) + + # scan + subdirs, rrds = self.scan_dir(path) + + # get top N hosts + hosts = backend.calc_top_hosts(path, rrds, 'in', 'out', count=count) + + # draw graph with hosts + w, h, data, img = graph.graph_multi('hourly', + "Top %d hosts" % count, + [('%s/%s.rrd' % (path, name), name) for name in hosts], + self.rrd_graph_func, + None # to tmpfile + ) + + # wrap file output + response_file = werkzeug.wrap_file(req.environ, img) + + return Response(response_file, mimetype='image/png', direct_passthrough=True) # map URLs to various methods # XXX: this uses the method object as the endpoint, which is a bit silly, since it's not bound and we need to pass # in self explicitly.. URLS = Map(( Rule('/', endpoint=index, defaults=dict(dir = '')), + Rule('/top.png', endpoint=graph_top, defaults=dict(dir = '')), Rule('//', endpoint=index), + Rule('//top.png', endpoint=graph_top), Rule('/.rrd', endpoint=target), Rule('/.rrd/.png', endpoint=graph, defaults=dict(interval = 'daily')), Rule('/.rrd//.png', endpoint=graph), diff -r cd9ca8068b09 -r 47e977c23ba2 wsgi-dev.py --- a/wsgi-dev.py Tue Jan 25 01:19:40 2011 +0200 +++ b/wsgi-dev.py Tue Jan 25 01:28:06 2011 +0200 @@ -1,10 +1,11 @@ +#!/usr/bin/env python """ Simple test server/environment for WSGI development """ import werkzeug -from rrdweb import wsgi +from rrdweb import wsgi, graph import logging @@ -13,9 +14,11 @@ logging.basicConfig(format="[%(levelname)5s] %(funcName)25s : %(message)s", level=logging.DEBUG) app = wsgi.WSGIApp( - rrdpath = 'rrd/', + rrdpath = 'var/rrd', tplpath = 'etc/templates', - imgpath = 'img/', + imgpath = 'var/img', + + rrdgraph = graph.pmacct_data, ) # run