# HG changeset patch # User Tero Marttila # Date 1288668976 -7200 # Node ID 9fa9d881fd8768344b1dbd140b611a1f2d01f59a # Parent 29a523db66a8619b86e7ae9e0a9ee2a6a09a111b# Parent 86bbabd10ff6bd21e0ef06b2559c4a0d92c5f7f9 Merge with previous bin/rrdweb development, and fix up merge bugs diff -r 29a523db66a8 -r 9fa9d881fd87 .hgignore --- a/.hgignore Tue Nov 02 05:09:09 2010 +0200 +++ b/.hgignore Tue Nov 02 05:36:16 2010 +0200 @@ -1,6 +1,7 @@ syntax: regexp # generated output +^www ^web/ ^img/ ^etc/mrtg-hosts.conf$ diff -r 29a523db66a8 -r 9fa9d881fd87 bin/rrdweb --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/rrdweb Tue Nov 02 05:36:16 2010 +0200 @@ -0,0 +1,214 @@ +#!/usr/bin/env python + +import os.path, fnmatch +import optparse, logging + +from rrdweb import graph, html + + +def parse_args (args) : + global options + + parser = optparse.OptionParser( + usage = "%prog [options] [...]" + ) + + # generic + parser.add_option('-q', "--quiet", help="No output in normal operation", + action='store_const', dest="loglvl", const=logging.WARNING, + ) + parser.add_option('-v', "--verbose", help="More output", + action='store_const', dest="loglvl", const=logging.INFO, + ) + parser.add_option('-D', "--debug", help="Even more output", + action='store_const', dest="loglvl", const=logging.DEBUG, + ) + + # options + parser.add_option( "--rrd-type", default="mrtg", help="RRD style to use: mrtg, collectd_ifoctets") + + # paths + parser.add_option('-R', "--rrd-dir", default="rrd", help="Path to directory containing .rrd files") + parser.add_option('-I', "--img-dir", default="img", help="Path to directory containing .png files") + parser.add_option('-W', "--web-dir", default="web", help="Path to directory containing .html files") + parser.add_option('-T', "--tpl-dir", default="templates", help="Path to directory containing .html templates files") + + # urls + parser.add_option('-P', "--url-prefix", default="", help="Prefix for URLs") + parser.add_option( "--img-url", default="img", help="URL to directory containing .png files") + parser.add_option( "--web-url", default="web", help="URL to directory containing .html files") + + # defaults + parser.set_defaults( + loglvl = logging.INFO, + ) + + ## parse + options, targets = parser.parse_args(args) + + + ## apply + logging.basicConfig( + format = "[%(levelname)8s] %(funcName)20s : %(message)s", + level = options.loglvl, + ) + + return targets + + +def scan_targets () : + """ + Scan for targets from rrd_dir + """ + + # root dir for searching + rrd_dir = options.rrd_dir + + # recurse + for dirpath, dirs, files in os.walk(rrd_dir) : + for name in files : + # full path + path = os.path.join(dirpath, name) + + # relative path from rrd_dir + relpath = path[len(os.path.commonprefix([rrd_dir, path])):] + + # skip dotfiles + if name.startswith('.') : + continue + + # foo/bar, .rrd + target, ext = os.path.splitext(relpath) + + if ext.lower() == '.rrd' : + # as target name + yield target + + + +def check_output_file (path) : + """ + Create the directory for the given output path if missing + """ + + dir, file = os.path.split(path) + + if not os.path.exists(dir) : + logging.warn("Create directory for output file %s: %s", file, dir) + + os.makedirs(dir) + +def target_graph (target, style, interval) : + """ + Generate .rrd -> .png for given target + """ + + # XXX: title.. + title = target + + # compose paths + rrd_path = os.path.join(options.rrd_dir, target) + '.rrd' + out_path = os.path.join(options.img_dir, style, interval, target) + '.png' + + # create out path if not exists + check_output_file(out_path) + + logging.debug("%s: %s -> %s", target, rrd_path, out_path) + + # graph function to use + graph_func = getattr(graph, options.rrd_type) + + # graph + graph_func(style, interval, title, rrd_path, out_path) + + +def target_html (target, formatter) : + """ + Generate .html for given target + """ + + # XXX: title.. + title = target + + # build paths + html_path = os.path.join(options.web_dir, target + '.html') + + # create out path if not exists + check_output_file(html_path) + + logging.debug("%s: %s", target, html_path) + + # render + open(html_path, 'w').write(formatter.target(target, title).encode('utf-8')) + +def overview_html (targets, formatter) : + """ + Generate .html index + """ + + # paths + overview_path = os.path.join(options.web_dir, "index.html") + + # create out path if not exists + check_output_file(overview_path) + + logging.debug("%s", overview_path) + + # as (target, title) pairs + open(overview_path, 'w').write(formatter.overview('/', ((target, target) for target in targets)).encode('utf-8')) + + +def generate_target (target, formatter) : + + # overview + target_graph(target, 'overview', 'daily') + + # details + for interval in ('daily', 'weekly', 'yearly') : + target_graph(target, 'detail', interval) + + # html for target + target_html(target, formatter) + + +def generate (targets, filters=None): + if not targets : + # autodetect + targets = sorted(list(scan_targets())) + + logging.info("Autodetected %d targets", len(targets)) + + if filters : + # filter targets + targets = [target for target in targets if any(fnmatch.fnmatch(target.name, filter) for filter in filters)] + + logging.info("Updating %d targets", len(targets)) + + # overview + formatter = html.Formatter( + url_prefix = options.url_prefix, + img_url = html.urljoin(options.url_prefix, options.img_url, "%(style)s", "%(interval)s", "%(target)s.png"), + target_url = html.urljoin(options.url_prefix, options.web_url, "%(target)s.html"), + ) + + # html for overview + overview_html(targets, formatter) + + for target in targets : + logging.info("Target: %s", target) + + # HTML and PNGs + generate_target(target, formatter) + +def main (args) : + # parse args + targets = parse_args(args) + + # run + generate(targets) + +if __name__ == '__main__' : + from sys import argv + + main(argv[1:]) + diff -r 29a523db66a8 -r 9fa9d881fd87 etc/generate.py --- a/etc/generate.py Tue Nov 02 05:09:09 2010 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,53 +0,0 @@ -from rrdweb import graph, html - -# list of targets -from targets import targets - -import os.path, fnmatch - -rrd_dir = "rrd" -img_dir = "img" -web_dir = "web" - -def target_graph (target, style, interval) : - # compose paths - rrd_path = os.path.join(rrd_dir, target.rrd_name()) - out_path = os.path.join(img_dir, style, interval, target.img_name()) - - # graph - graph.mrtg(style, interval, target.title, rrd_path, out_path) - - -def main (targets, filters, style='detail', interval='daily'): - if filters : - # filter targets - targets = [target for target in targets if any(fnmatch.fnmatch(target.name, filter) for filter in filters)] - - # overview - html_fmt = html.Formatter( - url_prefix = "/~terom/rrdweb", - img_url = "%(prefix)s/img/%(style)s/%(interval)s/%(target)s.png", - target_url = "%(prefix)s/web/%(target)s.html", - ) - - # overview page - overview_path = os.path.join(web_dir, "index.html") - open(overview_path, 'w').write(html_fmt.overview(targets)) - - for target in targets : - print target.name - - target_graph(target, 'overview', 'daily') - - for interval in ('daily', 'weekly', 'yearly') : - target_graph(target, 'detail', interval) - - # html - html_path = os.path.join(web_dir, target.name + '.html') - open(html_path, 'w').write(html_fmt.target(target)) - -if __name__ == '__main__' : - import sys - - main(targets, sys.argv[1:]) - diff -r 29a523db66a8 -r 9fa9d881fd87 rrdweb/graph.py --- a/rrdweb/graph.py Tue Nov 02 05:09:09 2010 +0200 +++ b/rrdweb/graph.py Tue Nov 02 05:36:16 2010 +0200 @@ -169,12 +169,35 @@ ] -def mrtg (style, interval, title, rrd_path, out_path) : +def collectd_data (rrd_path) : + """ + Data sources for if_octets from a collectd rrd + """ + + return [ + # data sources, bytes/s + r'DEF:in0=%s:rx:AVERAGE' % rrd_path, + r'DEF:out0=%s:tx:AVERAGE' % rrd_path, + + # data, bits/s + 'CDEF:in=in0,8,*', + 'CDEF:out=out0,8,*', + + ] + +def _graph (style, interval, title, data_func, rrd_path, out_path) : style_opts, style_vars = STYLE_DEFS[style]() interval_opts = INTERVAL_DEFS[interval](title) opts = rrd.merge_opts(common_opts(), style_opts, interval_opts) - data = mrtg_data(rrd_path) + style_vars + data = data_func(rrd_path) + style_vars return rrd.graph(out_path, *data, **opts) + +def mrtg (style, interval, title, rrd_path, out_path) : + return _graph(style, interval, title, mrtg_data, rrd_path, out_path) + +def collectd_ifoctets (style, interval, title, rrd_path, out_path) : + return _graph(style, interval, title, collectd_data, rrd_path, out_path) + diff -r 29a523db66a8 -r 9fa9d881fd87 rrdweb/html.py --- a/rrdweb/html.py Tue Nov 02 05:09:09 2010 +0200 +++ b/rrdweb/html.py Tue Nov 02 05:36:16 2010 +0200 @@ -5,6 +5,18 @@ import os.path +def urljoin (*parts) : + url = "" + + for part in parts : + if part : + if url : + url += "/" + part + else : + url += part + + return url + class BaseFormatter (object) : """ Trivial HTML template formatter. @@ -58,12 +70,16 @@ Format page contents """ - return self.render('layout', content=content) + return self.render('layout', + title = 'MRTG', + breadcrumb = '', # XXX: not supported + content = content + ) def fmt_img_url (self, style, interval, target) : return self.img_url % dict( prefix = self.url_prefix, - target = target.name, + target = target, style = style, interval = interval, ) @@ -71,33 +87,34 @@ def fmt_target_url (self, target) : return self.target_url % dict( prefix = self.url_prefix, - target = target.name, + target = target, ) - def overview (self, targets) : + def overview (self, dirname, targets) : """ - Format target listing + Format target listing for given list of (target, title) pairs """ return self.page(self.render('overview', - overview_graphs = '\n'.join( + dir = dirname, + overview_subdirs = '', # XXX: not supported + overview_graphs = '\n'.join( self.render('overview-target', - title = target.title, + title = title, daily_overview_img = self.fmt_img_url('overview', 'daily', target), target_url = self.fmt_target_url(target), - ) for target in targets - ) + ) for target, title in targets + ), )) - def target (self, target) : + def target (self, target, title) : """ Format a specific target """ return self.page(self.render('target', - title = target.title, - + title = title, daily_img = self.fmt_img_url('detail', 'daily', target), weekly_img = self.fmt_img_url('detail', 'weekly', target), yearly_img = self.fmt_img_url('detail', 'yearly', target), diff -r 29a523db66a8 -r 9fa9d881fd87 test.py --- a/test.py Tue Nov 02 05:09:09 2010 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,89 +0,0 @@ -from rrdweb import rrd - -rrd_path = "rrd/armo.switches.pvl_atk-luokka.rrd" -out_path = "img/armo.switches.pvl_atk-luokka.png" - -detail_size = (600, 200) - -# common options for all output -detail_opts = dict( - # output - imgformat = "PNG", - lazy = True, - - # dimensions - width = detail_size[0], - height = detail_size[1], - - color = [ - # disable border - # border = 0, - "SHADEA#ffffff00", - "SHADEB#ffffff00", - - # keep background transparent - "BACK#ffffff00", - "SHADEB#ffffff00", - ], - - # labels - vertical_label = "bits/s", - units = "si", - - # use logarithmic scaling - logarithmic = True, - - # smooth out lines - slope_mode = True, -) - -graph_opts = dict( - daily = dict( - # labels - x_grid = "MINUTE:15:HOUR:1:HOUR:4:0:%H:%M", - - # general info - title = "Daily Traffic @ armo.switches.pvl -> atk-luokka", - - # interval - start = "-24h", - ), -) - - - -rrd.graph(out_path, - # data sources, bytes/s - r'DEF:in0=%s:ds0:AVERAGE' % rrd_path, - r'DEF:out0=%s:ds1:AVERAGE' % rrd_path, - - # data, bits/s - 'CDEF:in=in0,8,*', - 'CDEF:out=out0,8,*', - - # values - 'VDEF:in_max=in,MAXIMUM', - 'VDEF:in_avg=in,AVERAGE', - 'VDEF:in_min=in,MINIMUM', - 'VDEF:out_max=out,MAXIMUM', - 'VDEF:out_avg=out,AVERAGE', - 'VDEF:out_min=out,MINIMUM', - - # legend/graph - "COMMENT:%4s" % "", - "COMMENT:%11s" % "Maximum", - "COMMENT:%11s" % "Average", - "COMMENT:%11s\\l" % "Minimum", - - "LINE1:in#0000FF:%4s" % "In", - 'GPRINT:in_max:%6.2lf %Sbps', - 'GPRINT:in_avg:%6.2lf %Sbps', - 'GPRINT:in_min:%6.2lf %Sbps\\l', - - "LINE1:out#00CC00:%4s" % "Out", - 'GPRINT:out_max:%6.2lf %Sbps', - 'GPRINT:out_avg:%6.2lf %Sbps', - 'GPRINT:out_min:%6.2lf %Sbps\\l', - - -)