--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/rrdweb/backend.py Tue Jan 25 01:28:06 2011 +0200
@@ -0,0 +1,92 @@
+from rrdweb import rrd
+
+import operator
+
+def host_def (idx, name, dir, ds_in, ds_out, cf) :
+ """
+ VDEFs for given host, giving its avg/max values:
+
+ DEF:in/out_{idx}
+ CDEFtraf_{idx}
+ VDEF:avg/max_{idx}
+ PRINT:{idx} avg/max {value}
+ """
+
+ params = dict(
+ dir=dir, ds_in=ds_in, ds_out=ds_out, cf=cf,
+ idx=idx, rrd='%s/%s.rrd' % (dir, name)
+ )
+
+ return [
+ # in/out bandwidth in bytes/s
+ 'DEF:in_%(idx)d=%(rrd)s:%(ds_in)s:%(cf)s' % params,
+ 'DEF:out_%(idx)d=%(rrd)s:%(ds_out)s:%(cf)s' % params,
+
+ # total traffic in bits/s
+ 'CDEF:traf_%(idx)d=in_%(idx)d,out_%(idx)d,+,8,*' % params,
+
+ # average + maximum values
+ 'VDEF:avg_%(idx)d=traf_%(idx)d,AVERAGE' % params,
+ 'VDEF:max_%(idx)d=traf_%(idx)d,MAXIMUM' % params,
+
+ # output
+ 'PRINT:avg_%(idx)d:%(idx)d avg %%lf' % params,
+ 'PRINT:max_%(idx)d:%(idx)d max %%lf' % params,
+ ]
+
+def parse_report (rrds, lines) :
+ """
+ Parse the report output into a [ (name, avg, max) ] list
+ """
+
+ # idx values by type
+ data = {
+ 'avg': {},
+ 'max': {},
+ }
+
+ # interpret output
+ for line in lines :
+ # parse
+ idx, type, value = line.split()
+
+ # convert
+ idx = int(idx)
+ value = float(value)
+
+ # store
+ data[type][idx] = value
+
+ # build into name : (values) dict
+ return [
+ (
+ (name, data['avg'][idx], data['max'][idx])
+ ) for idx, name in enumerate(rrds)
+ ]
+
+def calc_top_hosts (dir, rrds, ds_in, ds_out, period='15m', count=5, cf='AVERAGE') :
+ """
+ Return the list of top-N rrd's in the given dir, sorted by total in/out bandwidth average over the given period.
+ """
+
+ # vdefs for hosts, avg/max_<idx> in bits/s + prints
+ defs = [stmt for idx, name in enumerate(rrds) for stmt in host_def(idx, name, dir, ds_in, ds_out, cf)]
+
+ # execute
+ _, _, output, _ = rrd.graph(False, *defs, start=('-%s' % period))
+
+ # parse
+ data = parse_report(rrds, output)
+
+ # sort for top-N avg
+ data.sort(key=operator.itemgetter(1))
+ top_avg = data[:count]
+
+ # sort for top-N max
+ data.sort(key=operator.itemgetter(2))
+ top_max = data[:count]
+
+ # merge hosts (we lose sorting order)
+ return set(host for host, avg, max in top_avg + top_max)
+
+
--- a/rrdweb/graph.py Tue Jan 25 01:19:40 2011 +0200
+++ b/rrdweb/graph.py Tue Jan 25 01:28:06 2011 +0200
@@ -42,21 +42,21 @@
def overview_opts () :
"""
- Common options for the overview graph
+ Graph statements for a single-source overview graph
"""
return dict(
width = 600,
height = 50,
), [
- "CDEF:all=in,out,+",
+ "CDEF:all=in0,out0,+",
"VDEF:max=all,MAXIMUM",
"VDEF:avg=all,AVERAGE",
"VDEF:min=all,MINIMUM",
- "LINE1:in#0000FF:In",
- "LINE1:out#00CC00:Out",
+ "LINE1:in0#0000FF:In",
+ "LINE1:out0#00CC00:Out",
"GPRINT:max:%6.2lf %Sbps max",
"GPRINT:avg:%6.2lf %Sbps avg",
@@ -65,7 +65,7 @@
def detail_opts () :
"""
- Common options for the detail graph
+ Common options for a single-source detail graph
"""
return dict(
@@ -74,14 +74,14 @@
height = 200,
), [
# values
- 'VDEF:in_max=in,MAXIMUM',
- 'VDEF:in_avg=in,AVERAGE',
- 'VDEF:in_min=in,MINIMUM',
- 'VDEF:in_cur=in,LAST',
- 'VDEF:out_max=out,MAXIMUM',
- 'VDEF:out_avg=out,AVERAGE',
- 'VDEF:out_min=out,MINIMUM',
- 'VDEF:out_cur=out,LAST',
+ 'VDEF:in_max=in0,MAXIMUM',
+ 'VDEF:in_avg=in0,AVERAGE',
+ 'VDEF:in_min=in0,MINIMUM',
+ 'VDEF:in_cur=in0,LAST',
+ 'VDEF:out_max=out0,MAXIMUM',
+ 'VDEF:out_avg=out0,AVERAGE',
+ 'VDEF:out_min=out0,MINIMUM',
+ 'VDEF:out_cur=out0,LAST',
# legend/graph
"COMMENT:%4s" % "",
@@ -90,13 +90,13 @@
"COMMENT:%11s" % "Minimum",
"COMMENT:%11s\\l" % "Current",
- "LINE1:in#0000FF:%4s" % "In",
+ "LINE1:in0#0000FF:%4s" % "In",
'GPRINT:in_max:%6.2lf %Sbps',
'GPRINT:in_avg:%6.2lf %Sbps',
'GPRINT:in_min:%6.2lf %Sbps',
'GPRINT:in_cur:%6.2lf %Sbps\\l',
- "LINE1:out#00CC00:%4s" % "Out",
+ "LINE1:out0#00CC00:%4s" % "Out",
'GPRINT:out_max:%6.2lf %Sbps',
'GPRINT:out_avg:%6.2lf %Sbps',
'GPRINT:out_min:%6.2lf %Sbps',
@@ -106,11 +106,84 @@
"COMMENT:Generated %s\\r" % timestamp().replace(':', '\\:'),
]
+# set of line colors used for a multi-source graph, in rgb hex form (no prefix)
+MULTI_COLORS = (
+ '00cc00',
+ '0000ff',
+ 'cc0000',
+)
+
+def multi_graph_source_defs (idx, name, color) :
+ """
+ Render graph and legend summary for a single source in a multi-source graph.
+ """
+
+ params = dict(
+ idx = idx, name = name, color = color
+ )
+
+ return [stmt.format(**params) for stmt in (
+ "CDEF:all{idx}=in{idx},out{idx},+",
+
+ "VDEF:max{idx}=all{idx},MAXIMUM",
+ "VDEF:avg{idx}=all{idx},AVERAGE",
+ "VDEF:cur{idx}=all{idx},LAST",
+
+ "LINE1:all{idx}#{color}: ",
+
+ "GPRINT:max{idx}:%6.2lf %Sbps",
+ "GPRINT:avg{idx}:%6.2lf %Sbps",
+ "GPRINT:cur{idx}:%6.2lf %Sbps",
+ "COMMENT:\t{name}\\l",
+ )]
+
+
+def multi_graph (sources) :
+ """
+ Graph definition for a multi-source overview graph.
+
+ Uses combined in/out totals, giving a single line per source.
+ """
+
+ # defs for each source
+ sources_defs = [multi_graph_source_defs(idx, name, color) for (idx, name), color in zip(sources, MULTI_COLORS)]
+
+ return dict(
+ # dimensions
+ width = 600,
+ height = 200,
+
+ ), [
+ # legend header
+ "COMMENT: ",
+ "COMMENT:%11s" % "Maximum",
+ "COMMENT:%11s" % "Average",
+ "COMMENT:%11s" % "Current",
+ "COMMENT:\t%s\\l" % "",
+
+ ] + [
+ # each source
+ stmt for defs in sources_defs for stmt in defs
+ ]
+
STYLE_DEFS = {
'overview': overview_opts,
'detail': detail_opts,
}
+def hourly_opts (title) :
+
+ return dict(
+ # labels
+ x_grid = None,
+
+ # general info
+ title = "Hourly %s" % (title, ),
+
+ # interval
+ start = "-1h",
+ )
+
def daily_opts (title) :
"""
Common options for the 'daily' view
@@ -153,52 +226,99 @@
INTERVAL_DEFS = {
+ 'hourly': hourly_opts,
'daily': daily_opts,
'weekly': weekly_opts,
'yearly': yearly_opts,
}
-def mrtg_data (rrd_path) :
+def data_defs (idx, rrd, ds_in, ds_out, bytes=True, cf='AVERAGE') :
"""
- Data sources for network in/out from an MRTG rrd
+ Generate the DEF/CDEF statements for the in{idx}/out{idx} data sources for the given RRD and DS names.
+
+ If bytes is given, convert the value into bits.
"""
- return [
+ params = dict(
+ ds_in=ds_in, ds_out=ds_out, cf=cf,
+ idx=idx, rrd=rrd,
+ )
+
+ if bytes :
# data sources, bytes/s
- r'DEF:in0=%s:ds0:AVERAGE' % rrd_path,
- r'DEF:out0=%s:ds1:AVERAGE' % rrd_path,
-
+ yield 'DEF:in_raw{idx}={rrd}:{ds_in}:{cf}'.format(**params)
+ yield 'DEF:out_raw{idx}={rrd}:{ds_out}:{cf}'.format(**params)
+
# data, bits/s
- 'CDEF:in=in0,8,*',
- 'CDEF:out=out0,8,*',
+ yield 'CDEF:in{idx}=in_raw{idx},8,*'.format(**params)
+ yield 'CDEF:out{idx}=out_raw{idx},8,*'.format(**params)
+
+ else :
+ # data sources, bits/s
+ yield 'DEF:in{idx}={rrd}:{ds_in}:{cf}'.format(**params)
+ yield 'DEF:out{idx}={rrd}:{ds_out}:{cf}'.format(**params)
- ]
-def collectd_data (rrd_path) :
+def mrtg_data (idx, rrd) :
+ """
+ Generate the in{idx}/out{idx} data sources fro the given MRTG rrd.
+ """
+
+ return data_defs(idx, rrd, 'ds0', 'ds1')
+
+def collectd_data (idx, rrd) :
"""
Data sources for if_octets from a collectd rrd
"""
-
- return [
- # data sources, bytes/s
- r'DEF:in0=%s:rx:AVERAGE' % rrd_path,
- r'DEF:out0=%s:tx:AVERAGE' % rrd_path,
+
+ return data_defs(idx, rrd, 'rx', 'tx')
- # data, bits/s
- 'CDEF:in=in0,8,*',
- 'CDEF:out=out0,8,*',
+def pmacct_data (idx, rrd) :
+ """
+ Data sources for in/out bytes from a pmacct rrd
+ """
+
+ return data_defs(idx, rrd, 'in', 'out')
- ]
-
-def _graph (style, interval, title, data_func, rrd_path, out_path) :
+def graph_single (style, interval, title, data_func, rrd_path, out_path) :
+ """
+ Render graph.
+
+ Returns a (width, height, print_lines, graph_file) tuple.
+ """
+
style_opts, style_vars = STYLE_DEFS[style]()
interval_opts = INTERVAL_DEFS[interval](title)
opts = rrd.merge_opts(common_opts(), style_opts, interval_opts)
- data = data_func(rrd_path) + style_vars
+ data = list(data_func(0, rrd_path)) + style_vars
return rrd.graph(out_path, *data, **opts)
+def graph_multi (interval, title, rrd_list, data_func, out) :
+ """
+ Render a multi-source graph.
+
+ interval - the name of the time interval to use
+ title - graph title
+ rrd_list - sequence of (rrd, name) tuples for each source to draw
+ data_func - the data source definition to use
+ out - output path for graph
+ """
+
+ # data sources
+ data_defs = [stmt for idx, (path, name) in enumerate(rrd_list) for stmt in data_func(idx, path)]
+
+ # options
+ graph_opts, graph_defs = multi_graph([(idx, name) for idx, (path, name) in enumerate(rrd_list)])
+ interval_opts = INTERVAL_DEFS[interval](title)
+
+ # combine
+ opts = rrd.merge_opts(common_opts(), graph_opts, interval_opts)
+ defs = data_defs + graph_defs
+
+ # graph
+ return rrd.graph(out, *defs, **opts)
def mrtg (style, interval, title, rrd_path, out_path) :
return _graph(style, interval, title, mrtg_data, rrd_path, out_path)
@@ -206,3 +326,7 @@
def collectd_ifoctets (style, interval, title, rrd_path, out_path) :
return _graph(style, interval, title, collectd_data, rrd_path, out_path)
+def pmacct_bytes (style, interval, title, rrd_path, out_path) :
+ return _graph(style, interval, title, pmacct_data, rrd_path, out_path)
+
+
--- a/rrdweb/rrd.py Tue Jan 25 01:19:40 2011 +0200
+++ b/rrdweb/rrd.py Tue Jan 25 01:28:06 2011 +0200
@@ -4,7 +4,9 @@
import rrdtool
+import tempfile
import logging
+import os
log = logging.getLogger('rrdweb.rrd')
@@ -89,16 +91,58 @@
return func(*args)
-def graph (out_path, *args, **opts) :
+def graph (out, *args, **opts) :
"""
- Create a graph from data stored in one or several RRDs.
+ Render a graph image and/or print a report from data stored in one or several RRDs.
- Graph image output is written to the given path.
+ If the output path is given, the graph data is written to the given file, otherwise a temporary file is
+ created. The resulting output file is returned as an opened file object, ready to read() the graph data.
- Returns... something to do with the image's dimensions, or even the data itself?
+ In the temporary file case, the returned file is unlinked before being returned.
+
+ If out is explicitly given as False, no graph output is generated, and graph_file is returned as None.
+
+ Returns a (width, height, [ report output lines ], graph_file) tuple.
"""
+
+ # open/tempfile
+ if out is None :
+ # tempfile
+ tmp_fd, out_path = tempfile.mkstemp('.png')
- return run_cmd(rrdtool.graph, (out_path, ), opts, args)
+ # disregard the tmp_fd, we re-open the file after it's been written out
+ os.close(tmp_fd)
+
+ elif out is False :
+ # no output
+ tmp_fd = None
+ out_path = ''
+
+ else :
+ # direct output
+ tmp_fd = None
+ out_path = out
+
+ # render
+ try :
+ # invoke
+ width, height, output = run_cmd(rrdtool.graph, (out_path, ), opts, args)
+
+ # return resulting file
+ if out_path :
+ out_file = open(out_path, 'r')
+
+ else :
+ # no graph output
+ out_file = None
+
+ return width, height, output, out_file
+
+ finally :
+ if tmp_fd :
+ # cleanup tempfile
+ # XXX: definately not portable to windows
+ os.unlink(out_path)
def create (rrd_path, *args, **opts) :
"""
--- a/rrdweb/wsgi.py Tue Jan 25 01:19:40 2011 +0200
+++ b/rrdweb/wsgi.py Tue Jan 25 01:28:06 2011 +0200
@@ -7,7 +7,7 @@
from werkzeug import Request, Response
from werkzeug.routing import Map, Rule
-from rrdweb import html, graph
+from rrdweb import html, graph, backend
import os, os.path
import errno
@@ -19,20 +19,22 @@
class WSGIApp (object) :
- def __init__ (self, rrdpath, tplpath, imgpath) :
+ def __init__ (self, rrdpath, tplpath, imgpath, rrdgraph=graph.pmacct_bytes) :
"""
Configure
rrdpath - path to directory containing *.rrd files
tplpath - path to HTML templates
imgpath - path to generated PNG images. Must be writeable
+
+ rrdgraph - the graph.*_data function for rendering the graphs.
"""
self.rrdpath = os.path.abspath(rrdpath)
self.templates = html.BaseFormatter(tplpath)
+ self.imgpath = os.path.abspath(imgpath)
- # XXX: some kind of fancy cache thingie :)
- self.imgpath = os.path.abspath(imgpath)
+ self.rrd_graph_func = rrdgraph
# wrap to use werkzeug's Request/Response
@@ -79,6 +81,25 @@
return response
+ def fs_path (self, name) :
+ """
+ Lookup and return the full filesystem path to the given relative RRD file/dir.
+
+ The given name must be a relative path (no leading /).
+
+ Raises NotFound for invalid paths.
+ """
+
+ # full path
+ path = os.path.normpath(os.path.join(self.rrdpath, name))
+
+ # check inside base path
+ if not path.startswith(self.rrdpath) :
+ # not found
+ raise exceptions.NotFound(name)
+
+ # ok
+ return path
def scan_dir (self, dir) :
"""
@@ -253,29 +274,13 @@
return self.templates.render('target',
title = self.rrd_title(rrd),
+ hourly_img = url(self.graph, rrd=rrd, style='detail', interval='hourly'),
daily_img = url(self.graph, rrd=rrd, style='detail', interval='daily'),
weekly_img = url(self.graph, rrd=rrd, style='detail', interval='weekly'),
yearly_img = url(self.graph, rrd=rrd, style='detail', interval='yearly'),
)
- def fs_path (self, node) :
- """
- Lookup and return the full filesystem path to the given relative RRD/dir path.
- """
-
- # dir is relative (no leading slash)
- # full path
- path = os.path.normpath(os.path.join(self.rrdpath, node))
-
- # check inside base path
- if not path.startswith(self.rrdpath) :
- # mask
- raise exceptions.NotFound(node)
-
- # ok
- return path
-
def rrd_path (self, rrd) :
"""
Lookup and return the full filesystem path to the given RRD name.
@@ -300,7 +305,7 @@
def render_graph (self, rrd, style, interval, png_path) :
"""
- Render the given graph for the given RRD to the given path.
+ Render the given graph for the given RRD to the given path, returning the opened file object.
"""
rrd_path = self.rrd_path(rrd)
@@ -311,8 +316,11 @@
log.debug("%s -> %s", rrd_path, png_path)
- # XXX: always generate
- graph.mrtg(style, interval, title, rrd_path, png_path)
+ # generate using the variant function given
+ w, h, report, png_file = graph.graph_single(style, interval, title, self.rrd_graph_func, rrd_path, png_path)
+
+ # the open'd .tmp file
+ return png_file
def rrd_graph (self, rrd, style, interval, flush=False) :
"""
@@ -356,41 +364,32 @@
# re-generate to tmp file
tmp_path = os.path.join(self.imgpath, style, interval, rrd) + '.tmp'
-
- self.render_graph(rrd, style, interval, tmp_path)
+
+ # open and write the graph image
+ img_file = self.render_graph(rrd, style, interval, tmp_path)
# replace .png with .tmp (semi-atomic, but atomic enough..)
+ # XXX: probably not portable to windows, what with img_file
os.rename(tmp_path, img_path)
-
- # open the now-fresh .png and return that
- return open(img_path)
-
-
+
+ else :
+ # use existing file
+ img_file = open(img_path, 'rb')
+
+ # return the now-fresh .png and return that
+ return img_file
### Request handlers
-
-
-# def node (self, url, path) :
-# """
-# Arbitrate between URLs to dirs and to RRDs.
-# """
-#
-
-
def index (self, req, url, dir = '') :
"""
Directory overview
dir - (optional) relative path to subdir from base rrdpath
"""
-
- # lookup fs path
+
+ # lookup
path = self.fs_path(dir)
- # found?
- if not os.path.isdir(path) :
- raise exceptions.NotFound("No such RRD directory: %s" % (dir, ))
-
# scan
subdirs, rrds = self.scan_dir(path)
@@ -444,14 +443,42 @@
# respond with file wrapper
return Response(response_file, mimetype='image/png', direct_passthrough=True)
+
+ def graph_top (self, req, url, dir = '', count=5) :
+ """
+ Show top N hosts by peak/average.
+ """
+ # find
+ path = self.fs_path(dir)
+
+ # scan
+ subdirs, rrds = self.scan_dir(path)
+
+ # get top N hosts
+ hosts = backend.calc_top_hosts(path, rrds, 'in', 'out', count=count)
+
+ # draw graph with hosts
+ w, h, data, img = graph.graph_multi('hourly',
+ "Top %d hosts" % count,
+ [('%s/%s.rrd' % (path, name), name) for name in hosts],
+ self.rrd_graph_func,
+ None # to tmpfile
+ )
+
+ # wrap file output
+ response_file = werkzeug.wrap_file(req.environ, img)
+
+ return Response(response_file, mimetype='image/png', direct_passthrough=True)
# map URLs to various methods
# XXX: this uses the method object as the endpoint, which is a bit silly, since it's not bound and we need to pass
# in self explicitly..
URLS = Map((
Rule('/', endpoint=index, defaults=dict(dir = '')),
+ Rule('/top.png', endpoint=graph_top, defaults=dict(dir = '')),
Rule('/<path:dir>/', endpoint=index),
+ Rule('/<path:dir>/top.png', endpoint=graph_top),
Rule('/<path:rrd>.rrd', endpoint=target),
Rule('/<path:rrd>.rrd/<string:style>.png', endpoint=graph, defaults=dict(interval = 'daily')),
Rule('/<path:rrd>.rrd/<string:style>/<string:interval>.png', endpoint=graph),
--- a/wsgi-dev.py Tue Jan 25 01:19:40 2011 +0200
+++ b/wsgi-dev.py Tue Jan 25 01:28:06 2011 +0200
@@ -1,10 +1,11 @@
+#!/usr/bin/env python
"""
Simple test server/environment for WSGI development
"""
import werkzeug
-from rrdweb import wsgi
+from rrdweb import wsgi, graph
import logging
@@ -13,9 +14,11 @@
logging.basicConfig(format="[%(levelname)5s] %(funcName)25s : %(message)s", level=logging.DEBUG)
app = wsgi.WSGIApp(
- rrdpath = 'rrd/',
+ rrdpath = 'var/rrd',
tplpath = 'etc/templates',
- imgpath = 'img/',
+ imgpath = 'var/img',
+
+ rrdgraph = graph.pmacct_data,
)
# run