pvl.rrd.graph: refactor to use Graph -> Interface -> Mrtg/CollectdIfOctets
authorTero Marttila <terom@paivola.fi>
Sun, 20 Jan 2013 19:52:41 +0200
changeset 155 9f2967ba81ef
parent 154 11df86fd2d67
child 156 999ae3e9fdec
pvl.rrd.graph: refactor to use Graph -> Interface -> Mrtg/CollectdIfOctets
bin/pvl.verkko-rrd
pvl/rrd/api.py
pvl/rrd/graph.py
pvl/verkko/rrd.py
--- a/bin/pvl.verkko-rrd	Sun Jan 20 18:51:51 2013 +0200
+++ b/bin/pvl.verkko-rrd	Sun Jan 20 19:52:41 2013 +0200
@@ -32,6 +32,9 @@
     # common
     parser.add_option_group(pvl.args.parser(parser))
 
+    parser.add_option('--rrd-type', metavar='TYPE', default='collectd',
+        help="mrtg/collectd")
+
     parser.add_option('--rrd', metavar='PATH',
         help="Find RRD files")
 
@@ -46,6 +49,7 @@
 
     return options, args
 
+
 def main (argv) :
     """
         pvl.verkko wsgi development server.
@@ -55,11 +59,13 @@
     options, args = parse_argv(argv, doc=__doc__)
 
     # rrd
+    rrd_type = pvl.rrd.graph.interface(options.rrd_type)
+
     if not options.rrd :
         log.error("no --rrd given")
         return 2
 
-    rrd = pvl.verkko.rrd.RRDDatabase(options.rrd, options.cache)
+    rrd = pvl.verkko.rrd.RRDDatabase(rrd_type, options.rrd, options.cache)
 
     # app
     application = pvl.verkko.rrd.Application(rrd)
--- a/pvl/rrd/api.py	Sun Jan 20 18:51:51 2013 +0200
+++ b/pvl/rrd/api.py	Sun Jan 20 19:52:41 2013 +0200
@@ -45,7 +45,7 @@
 
     return func(*(pre + opts + post))
 
-def graph (out=None, *args, **opts) :
+def graph (out=None, *defs, **opts) :
     """
         Render a graph image and/or print a report from data stored in one or several RRDs.
         
@@ -80,7 +80,7 @@
     log.debug("%s", out_path)
 
     # XXX: handle tempfile close?
-    width, height, out_lines = cmd(rrdtool.graph, (out_path, ), opts, args)
+    width, height, out_lines = cmd(rrdtool.graph, (out_path, ), opts, defs)
 
     if out_file is True :
         out_file = open(out_path)
--- a/pvl/rrd/graph.py	Sun Jan 20 18:51:51 2013 +0200
+++ b/pvl/rrd/graph.py	Sun Jan 20 19:52:41 2013 +0200
@@ -4,206 +4,299 @@
 from pvl.invoke import merge # XXX
 
 """
-    RRDTool graph output for MRTG, or collectd if_octets
+    RRDTool graph builders.
+
+    Includes support for Smokeping-inspired interface graphs from MRTG/Collectd.
 """
 
 def timestamp () :
     return time.strftime("%Y/%m/%d %H:%M:%S %Z")
 
-def common_opts () :
-    """
-        Common options for all views
+class Graph (object) :
     """
-
-    return dict(
-        # output
-        imgformat           = "PNG",
-        #        lazy                = True,
-
-        color               = [
-            # disable border
-            # border            = 0,
-            "SHADEA#ffffff00",
-            "SHADEB#ffffff00",
-
-            # keep background transparent
-            "BACK#ffffff00",
-            "SHADEB#ffffff00",
-        ],
-         
-        # labels
-        vertical_label      = "bits/s",
-        units               = "si",
-
-        # use logarithmic scaling?
-        #        logarithmic         = True,
+        Render an RRD graph from definitions/options.
         
-        # smooth out lines
-        slope_mode          = True,
-    )
-
-def overview_opts () :
-    """
-        Common options for the overview graph
-    """
-
-    return dict(
-        width               = 600,
-        height              = 50,
-    ), [
-        "CDEF:all=in,out,+",
+        Acts as a semi-magical object, with immutable state, and methods that return updated copies of our state.
 
-        "VDEF:max=all,MAXIMUM",
-        "VDEF:avg=all,AVERAGE",
-        "VDEF:min=all,MINIMUM",
-
-        "LINE1:in#0000FF:In",
-        "LINE1:out#00CC00:Out",
-
-        "GPRINT:max:%6.2lf %Sbps max",
-        "GPRINT:avg:%6.2lf %Sbps avg",
-        "GPRINT:min:%6.2lf %Sbps min\\l",
-    ]
-
-def detail_opts () :
-    """
-        Common options for the detail graph
+        >>> Graph()
+        Graph()
+        >>> Graph('foo')
+        Graph('foo')
+        >>> Graph(bar='bar')
+        Graph(bar='bar')
+        >>> Graph('foo', bar='bar')
+        Graph('foo', bar='bar')
+        >>> Graph('foo')(bar='bar')
+        Graph('foo', bar='bar')
     """
 
-    return dict(
-        # dimensions
-        width               = 600,
-        height              = 200,
-    ), [
-        # values
-        'VDEF:in_max=in,MAXIMUM',
-        'VDEF:in_avg=in,AVERAGE',
-        'VDEF:in_min=in,MINIMUM',
-        'VDEF:in_cur=in,LAST',
-        'VDEF:out_max=out,MAXIMUM',
-        'VDEF:out_avg=out,AVERAGE',
-        'VDEF:out_min=out,MINIMUM',
-        'VDEF:out_cur=out,LAST',
-
-        # legend/graph
-        "COMMENT:%4s" % "",
-        "COMMENT:%11s" % "Maximum",
-        "COMMENT:%11s" % "Average",
-        "COMMENT:%11s" % "Minimum",
-        "COMMENT:%11s\\l" % "Current",
+    def __init__ (self, *defs, **opts) :
+        self.defs = defs
+        self.opts = opts
 
-        "LINE1:in#0000FF:%4s" % "In",
-        'GPRINT:in_max:%6.2lf %Sbps',
-        'GPRINT:in_avg:%6.2lf %Sbps',
-        'GPRINT:in_min:%6.2lf %Sbps',
-        'GPRINT:in_cur:%6.2lf %Sbps\\l',
+    def __call__ (self, *defs, **opts) :
+        return type(self)(*(self.defs + defs), **merge(self.opts, opts))
 
-        "LINE1:out#00CC00:%4s" % "Out",
-        'GPRINT:out_max:%6.2lf %Sbps',
-        'GPRINT:out_avg:%6.2lf %Sbps',
-        'GPRINT:out_min:%6.2lf %Sbps',
-        'GPRINT:out_cur:%6.2lf %Sbps\\l',
-        
-        # mark
-        "COMMENT:Generated %s\\r" % timestamp().replace(':', '\\:'),
-    ]
+    def __getitem__ (self, key) :
+        return self.opts[key]
 
-STYLE_DEFS = {
-    'overview': overview_opts,
-    'detail':   detail_opts,
-}
+    def graph (self, out) :
+        """
+            Render completed graph using pvl.rrd.api.graph()
+        """
 
-def daily_opts (title) :
+        return rrd.graph(out, *self.defs, **self.opts)
+    
+    def __repr__ (self) :
+        return "{type}({args})".format(
+                type    = self.__class__.__name__,
+                args    = ', '.join([repr(def_) for def_ in self.defs] + [str(opt) + '=' + repr(value) for opt, value in self.opts.iteritems()]),
+        )
+
+    # builders
+class Interface (Graph) :
     """
-        Common options for the 'daily' view
+        An RRD graph showing in/out traffic in bits/s on an interface, with style/interval support.
     """
 
-    return dict(
-        # labels
-        x_grid              = "MINUTE:15:HOUR:1:HOUR:4:0:%H:%M",
-    
-        # general info
-        title               = "Daily %s" % (title, ),
-    
-        # interval
-        start               = "-24h",
-    )
+    @classmethod
+    def build (cls, title, rrd, style, interval) :
+        """
+            Return a simple Graph using the given title, RRD and style/interval
 
-def weekly_opts (title) :
-    return dict(
-        # labels
-        #        x_grid              = "MINUTE:15:HOUR:1:HOUR:4:0:%H:%M",
-    
-        # general info
-        title               = "Weekly %s" % (title, ),
+                title       - common(title=...)
+                rrd         - data(rrd=...)
+                style       - style(style=...)
+                interval    - interval(interval=...)
+        """
+
+        graph = cls()
+        graph = graph.common(title)
+        graph = graph.source(rrd)
+        graph = graph.style(style)
+        graph = graph.interval(interval)
+
+        return graph
+
+    # builders
+    def common (self, title) :
+        """
+            Common options for all views
+        """
+
+        return self(
+            # output
+            imgformat           = "PNG",
+            #        lazy                = True,
+
+            color               = [
+                # disable border
+                # border            = 0,
+                "SHADEA#ffffff00",
+                "SHADEB#ffffff00",
+
+                # keep background transparent
+                "BACK#ffffff00",
+                "SHADEB#ffffff00",
+            ],
+             
+            # labels
+            vertical_label      = "bits/s",
+            title               = title,
+            units               = "si",
+
+            # use logarithmic scaling?
+            #        logarithmic         = True,
+            
+            # smooth out lines
+            slope_mode          = True,
+        )
     
-        # interval
-        start               = "-7d",
-    )
+    ## data
+    # DS names; in/out bytes
+    IN = OUT = None
 
-def yearly_opts (title) :
-    return dict(
-        # labels
-        #        x_grid              = "MINUTE:15:HOUR:1:HOUR:4:0:%H:%M",
+    def source (self, rrd) :
+        """
+            Abstract: rrd -> in/out
+        """
+
+        if not (self.IN and self.OUT) :
+            raise TypeError("No IN/OUT DS names for %s" % (self, ))
+
+        return self(
+            # data sources, bytes/s
+            r'DEF:in0={rrd}:{ds_in}:AVERAGE'.format(rrd=rrd, ds_in=self.IN),
+            r'DEF:out0={rrd}:{ds_out}:AVERAGE'.format(rrd=rrd, ds_out=self.OUT),
+
+            # data, bits/s
+            'CDEF:in=in0,8,*',
+            'CDEF:out=out0,8,*',
+        )
     
-        # general info
-        title               = "Yearly %s" % (title, ),
+    ## style
+    def style_overview (graph) :
+        """
+            in/out bps -> overview graph
+        """
+
+        return graph(
+            "CDEF:all=in,out,+",
+
+            "VDEF:max=all,MAXIMUM",
+            "VDEF:avg=all,AVERAGE",
+            "VDEF:min=all,MINIMUM",
+
+            "LINE1:in#0000FF:In",
+            "LINE1:out#00CC00:Out",
+
+            "GPRINT:max:%6.2lf %Sbps max",
+            "GPRINT:avg:%6.2lf %Sbps avg",
+            "GPRINT:min:%6.2lf %Sbps min\\l",
+
+            # dimensions
+            width               = 600,
+            height              = 50,
+        )
+
+    def style_detail (graph) :
+        """
+            in/out bps -> detail graph
+        """
+
+        return graph(
+            # values
+            'VDEF:in_max=in,MAXIMUM',
+            'VDEF:in_avg=in,AVERAGE',
+            'VDEF:in_min=in,MINIMUM',
+            'VDEF:in_cur=in,LAST',
+            'VDEF:out_max=out,MAXIMUM',
+            'VDEF:out_avg=out,AVERAGE',
+            'VDEF:out_min=out,MINIMUM',
+            'VDEF:out_cur=out,LAST',
+
+            # legend/graph
+            "COMMENT:%4s" % "",
+            "COMMENT:%11s" % "Maximum",
+            "COMMENT:%11s" % "Average",
+            "COMMENT:%11s" % "Minimum",
+            "COMMENT:%11s\\l" % "Current",
+
+            "LINE1:in#0000FF:%4s" % "In",
+            'GPRINT:in_max:%6.2lf %Sbps',
+            'GPRINT:in_avg:%6.2lf %Sbps',
+            'GPRINT:in_min:%6.2lf %Sbps',
+            'GPRINT:in_cur:%6.2lf %Sbps\\l',
+
+            "LINE1:out#00CC00:%4s" % "Out",
+            'GPRINT:out_max:%6.2lf %Sbps',
+            'GPRINT:out_avg:%6.2lf %Sbps',
+            'GPRINT:out_min:%6.2lf %Sbps',
+            'GPRINT:out_cur:%6.2lf %Sbps\\l',
+            
+            # mark
+            "COMMENT:Generated {now}\\r".format(now=timestamp().replace(':', '\\:')),
+
+            # dimensions
+            width               = 600,
+            height              = 200,
+        )
+
+    STYLE = {
+        'overview': style_overview,
+        'detail':   style_detail,
+    }
+
+    def style (self, style) :
+        return self.STYLE[style](self)
     
-        # interval
-        start               = "-1y",
-    )
+    ## interval
+    def interval_daily (graph) :
+        """
+            Common options for the 'daily' view
+        """
 
+        return graph(
+            # labels
+            x_grid              = "MINUTE:15:HOUR:1:HOUR:4:0:%H:%M",
+        
+            # general info
+            title               = "Daily {graph[title]}".format(graph=graph),
+        
+            # interval
+            start               = "-24h",
+        )
 
-INTERVAL_DEFS = {
-    'daily':    daily_opts,
-    'weekly':   weekly_opts,
-    'yearly':   yearly_opts,
+    def interval_weekly (graph) :
+        return graph(
+            # labels
+            #        x_grid              = "MINUTE:15:HOUR:1:HOUR:4:0:%H:%M",
+        
+            # general info
+            title               = "Weekly {graph[title]}".format(graph=graph),
+        
+            # interval
+            start               = "-7d",
+        )
+
+    def interval_yearly (graph) :
+        return graph(
+            # labels
+            #        x_grid              = "MINUTE:15:HOUR:1:HOUR:4:0:%H:%M",
+        
+            # general info
+            title               = "Yearly {graph[title]}".format(graph=graph),
+        
+            # interval
+            start               = "-1y",
+        )
+
+    INTERVAL = {
+        'daily':    interval_daily,
+        'weekly':   interval_weekly,
+        'yearly':   interval_yearly,
+    }
+
+    def interval (self, interval) :
+        return self.INTERVAL[interval](self)
+
+# specific types
+class Mrtg (Interface) :
+    """
+        MRTG -> in/out
+    """
+
+    IN = 'ds0'
+    OUT = 'ds1'
+
+class CollectdIfOctets (Interface) :
+    """
+        Collectd if_octets -> in/out
+    """
+
+    IN = 'rx'
+    OUT = 'tx'
+
+INTERFACE = {
+    'mrtg':         Mrtg,
+    'collectd':     CollectdIfOctets,
 }
 
-def mrtg_data (rrd_path) :
-    """
-        Data sources for network in/out from an MRTG rrd
+def interface (type) :
     """
-
-    return [
-        # data sources, bytes/s
-        r'DEF:in0=%s:ds0:AVERAGE' % rrd_path,
-        r'DEF:out0=%s:ds1:AVERAGE' % rrd_path,
-
-        # data, bits/s
-        'CDEF:in=in0,8,*',
-        'CDEF:out=out0,8,*',
-
-    ]        
-
-def collectd_data (rrd_path) :
-    """
-        Data sources for if_octets from a collectd rrd
+        Lookup Interface -subclass for given type.
     """
 
-    return [
-        # data sources, bytes/s
-        r'DEF:in0=%s:rx:AVERAGE' % rrd_path,
-        r'DEF:out0=%s:tx:AVERAGE' % rrd_path,
-
-        # data, bits/s
-        'CDEF:in=in0,8,*',
-        'CDEF:out=out0,8,*',
+    return INTERFACE[type]
 
-    ]        
-    
-def _graph (style, interval, title, data_func, rrd_path, out_path) :
-    style_opts, style_vars = STYLE_DEFS[style]()
-    interval_opts = INTERVAL_DEFS[interval](title)
+# XXX: legacy
+def mrtg (style, interval, title, rrd, out) :
+    return MrtgInterface.build(title, rrd, style, interval).graph(out)
 
-    opts = merge(common_opts(), style_opts, interval_opts)
-    data = data_func(rrd_path) + style_vars
-
-    return rrd.graph(out_path, *data, **opts)
+def collectd_ifoctets (style, interval, title, rrd, out) :
+    return CollectdIfOctets.build(title, rrd, style, interval).graph(out)
 
-def mrtg (style, interval, title, rrd_path, out_path) :
-    return _graph(style, interval, title, mrtg_data, rrd_path, out_path)
+if __name__ == '__main__':
+    import doctest
+    doctest.testmod()
 
-def collectd_ifoctets (style, interval, title, rrd_path, out_path) :
-    return _graph(style, interval, title, collectd_data, rrd_path, out_path)
-
--- a/pvl/verkko/rrd.py	Sun Jan 20 18:51:51 2013 +0200
+++ b/pvl/verkko/rrd.py	Sun Jan 20 19:52:41 2013 +0200
@@ -19,12 +19,19 @@
         A filesystem directory containing .rrd files.
     """
 
-    def __init__ (self, path, cache=None) :
+    def __init__ (self, graph, path, cache=None) :
+        """
+            graph   - pvl.rrd.graph.InterfaceGraph type
+            path    - path to rrd dirs
+            cache   - path to cache dirs
+        """
+
         if not path :
             raise ValueError("RRDDatabase: no path given")
 
-        log.info("%s: cache=%s", path, cache)
-
+        log.info("%s: type=%s, cache=%s", path, graph, cache)
+        
+        self._graph = graph
         self._path = path
         self._cache = cache
 
@@ -177,9 +184,7 @@
 
         else :
             # to cache
-            # XXX: lookup graph style..
-            # XXX: collectd
-            dimensions, lines, outfile = pvl.rrd.graph.collectd_ifoctets(style, interval, title, path, out)
+            dimensions, lines, outfile = self._graph.build(title, path, style, interval).graph(out)
 
         return outfile