move pvl.verkko.dhcp and pvl.syslog.dhcp to pvl.dhcp.leases/syslog
authorTero Marttila <terom@paivola.fi>
Sat, 26 Jan 2013 11:09:30 +0200
changeset 169 a81ca751664d
parent 168 4e120851ff52
child 170 455573d46a33
move pvl.verkko.dhcp and pvl.syslog.dhcp to pvl.dhcp.leases/syslog
bin/pvl.syslog-dhcp
pvl/dhcp/__init__.py
pvl/dhcp/leases.py
pvl/dhcp/syslog.py
pvl/syslog/dhcp.py
pvl/verkko/dhcp/__init__.py
pvl/verkko/dhcp/leases.py
--- a/bin/pvl.syslog-dhcp	Fri Jan 25 22:02:55 2013 +0200
+++ b/bin/pvl.syslog-dhcp	Sat Jan 26 11:09:30 2013 +0200
@@ -10,8 +10,8 @@
 import pvl.syslog.args
 
 import pvl.verkko.db as db
-import pvl.syslog.dhcp
-import pvl.verkko.dhcp.leases
+import pvl.dhcp.syslog
+import pvl.dhcp.leases
 
 import logging, optparse
 
@@ -294,7 +294,7 @@
         self.leasedb = DHCPLeasesDatabase(db)
 
         # XXX
-        self.filter = pvl.syslog.dhcp.DHCPSyslogFilter()
+        self.parser = pvl.dhcp.syslog.DHCPSyslogParser()
 
     def createdb (self) :
         """
@@ -309,7 +309,7 @@
             Handle a single item read from syslog to DB.
         """
 
-        dhcp_item = self.filter.parse(item['msg'])
+        dhcp_item = self.parser.parse(item['msg'])
 
         log.debug("%s: %s", item, dhcp_item)
         
@@ -360,15 +360,10 @@
             
             if self.leases :
                 # process internally
-                #sync, leases = self.leases.process()
                 leases = self.leases.process()
 
-                #if sync :
-                #    self.sync_leases(leases)
                 if leases :
                     self.process_leases(leases)
-                #else :
-                #    pass
 
             if not poll :
                 # done
@@ -408,7 +403,7 @@
     # leases
     if options.leases_file :
         log.info("Open up DHCP leases...")
-        leases = pvl.verkko.dhcp.leases.DHCPLeasesDatabase(options.leases_file)
+        leases = pvl.dhcp.leases.DHCPLeasesDatabase(options.leases_file)
 
         # force polling interval
         if options.leases_tail :
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pvl/dhcp/leases.py	Sat Jan 26 11:09:30 2013 +0200
@@ -0,0 +1,450 @@
+"""
+    DHCP dhcpd.leases handling/tracking
+"""
+
+import pvl.syslog.tail # TailFile
+
+from datetime import datetime
+
+import logging; log = logging.getLogger('pvl.dhcp.leases')
+
+class DHCPLeasesParser (object) :
+    """
+        Simplistic parser for a dhcpd.leases file.
+
+        Doesn't implement the full spec, but a useful approximation.
+    """
+
+    def __init__ (self) :
+        self.block = None
+        self.items = []
+    
+    @classmethod
+    def split (cls, line) :
+        """
+            Split given line-data.
+            
+            >>> split = DHCPLeasesParser.split
+            >>> split('foo bar')
+            ['foo', 'bar']
+            >>> split('"foo"')
+            ['foo']
+            >>> split('foo "asdf quux" bar')
+            ['foo', 'asdf quux', 'bar']
+            >>> split('foo "asdf quux"')
+            ['foo', 'asdf quux']
+        """
+
+        # parse out one str
+        if '"' in line :
+            log.debug("%s", line)
+
+            # crude
+            pre, line = line.split('"', 1)
+            data, post = line.rsplit('"', 1)
+
+            return pre.split() + [data] + post.split()
+        else :
+            return line.split()
+
+    @classmethod
+    def lex (self, line) :
+        """
+            Yield tokens from the given lines.
+
+            >>> lex = DHCPLeasesParser.lex
+            >>> list(lex('foo;'))
+            [('item', ['foo'])]
+            >>> list(item for line in ['foo {', ' bar;', '}'] for item in lex(line))
+            [('open', ['foo']), ('item', ['bar']), ('close', None)]
+
+        """
+
+        log.debug("%s", line)
+
+        # comments?
+        if '#' in line :
+            line, comment = line.split('#', 1)
+        else :
+            comment = None
+
+        # clean?
+        line = line.strip()
+
+        # parse
+        if not line :
+            # ignore, empty/comment
+            return
+        
+        elif line.startswith('uid') :
+            # XXX: too hard to parse properly
+            return
+
+        elif '{' in line :
+            decl, line = line.split('{', 1)
+
+            # we are in a new decl
+            yield 'open', self.split(decl)
+       
+        elif ';' in line :
+            param, line = line.split(';', 1)
+            
+            # a stanza
+            yield 'item', self.split(param)
+        
+        elif '}' in line :
+            close, line = line.split('}', 1)
+
+            if close.strip() :
+                log.warn("Predata on close: %s", close)
+
+            # end
+            yield 'close', None
+    
+        else :
+            log.warn("Unknown line: %s", line)
+            return
+
+        # got the whole line?
+        if line.strip() :
+            log.warn("Data remains: %s", line)
+
+    def push_block (self, block) :
+        """
+            Open new block.
+        """
+        
+        # XXX: stack
+        assert not self.block
+
+        self.block = block
+        self.items = []
+
+    def feed_block (self, item) :
+        """
+            Add item to block
+        """
+
+        assert self.block
+
+        self.items.append(item)
+
+    def pop_block (self) :
+        """
+            Close block. Returns
+                (block, [items])
+        """
+
+        assert self.block
+
+        block, items = self.block, self.items
+
+        self.block = None
+        self.items = None
+
+        return block, items
+
+    def parse (self, line) :
+        """
+            Parse given line, yielding any complete blocks that come out.
+
+            >>> parser = DHCPLeasesParser()
+            >>> list(parser.parse_lines(['foo {', ' bar;', ' quux asdf;', '}']))
+            [(['foo'], [['bar'], ['quux', 'asdf']])]
+
+            >>> parser = DHCPLeasesParser()
+            >>> list(parser.parse('foo {'))
+            []
+            >>> list(parser.parse_lines([' bar;', ' quux asdf;']))
+            []
+            >>> list(parser.parse('}'))
+            [(['foo'], [['bar'], ['quux', 'asdf']])]
+        """
+
+        for token, args in self.lex(line) :
+            #log.debug("%s: %s [block=%s]", token, args, self.block)
+
+            if token == 'open' :
+                # open new block
+                block = args
+
+                if self.block :
+                    log.warn("nested blocks: %s > %s", self.block, block)
+                    continue
+            
+                log.debug("open block: %s", block)
+                self.push_block(block)
+            
+            elif token == 'close' :
+                log.debug("close block: %s", self.block)
+
+                # collected block items
+                yield self.pop_block()
+
+            # must be within block!
+            elif token == 'item' :
+                item = args
+
+                if not self.block :
+                    log.warn("token outside block: %s: %s", token, args)
+                    continue
+
+                log.debug("block %s item: %s", self.block, item)
+                self.feed_block(item)
+
+            else :
+                # ???
+                raise KeyError("Unknown token: {0}: {1}".format(token, args))
+    
+    def parse_lines (self, lines) :
+        """
+            Trivial wrapper around parse to parse multiple lines.
+        """
+
+        for line in lines :
+            for item in self.parse(line) :
+                yield item
+
+class DHCPLeasesDatabase (object) :
+    """
+        Process log-structured leases file.
+    """
+
+    LEASE_DATES = ('starts', 'ends', 'tstp', 'tsfp', 'atsfp', 'cltt')
+
+    # default format
+    LEASE_DATE_NEVER = 'never'
+    LEASE_DATE_FMT_DEFAULT = '%w %Y/%m/%d %H:%M:%S'
+
+    lease_date_fmt = LEASE_DATE_FMT_DEFAULT
+
+    def __init__ (self, path) :
+        """
+            path        - path to dhcpd.leases file
+        """
+
+        # tail; handles file re-writes
+        self.source = pvl.syslog.tail.TailFile(path)
+
+        # parser state
+        self.parser = DHCPLeasesParser()
+
+        # initial leases state
+        self.leases = None
+
+    def reset (self) :
+        """
+            Reset state, if we started to read a new file.
+        """
+
+        self.leases = {}
+
+    def process_lease_item_date (self, args) :
+        """
+            Process lease-item date spec into datetime.
+
+            Returns None if 'never'.
+        """
+
+        data = ' '.join(args)
+
+        if data == self.LEASE_DATE_NEVER :
+            return None
+        else :
+            return datetime.strptime(data, self.lease_date_fmt)
+
+    def process_lease_item (self, lease, item) :
+        """
+            Process a single item from the lease, updating the lease dict
+        """
+
+        item = list(item)
+
+        name = item.pop(0)
+        subname = item[0] if item else None
+
+        if name in self.LEASE_DATES :
+            lease[name] = self.process_lease_item_date(item)
+
+        elif name == 'hardware':
+            # args
+            lease['hwtype'], lease['hwaddr'] = item
+        
+        elif name == 'uid' :
+            lease['uid'], = item
+
+        elif name == 'client-hostname' :
+            lease['client-hostname'], = item
+        
+        elif name == 'abandoned' :
+            lease['abandoned'] = True
+
+        elif name == 'binding' :
+            _state, lease['binding-state'] = item
+
+        elif name == 'next' and subname == 'binding' :
+            _binding, _state, lease['next-binding-state'] = item
+
+        else :
+            log.warn("unknown lease item: %s: %s", name, item)
+
+    def process_lease (self, lease_name, items) :
+        """
+            Process given lease block to update our state.
+
+            Returns the lease object, and a possible old lease.
+        """
+
+        # replace any existing
+        lease = self.leases[lease_name] = {}
+
+        # meta
+        lease['lease'] = lease_name
+
+        # parse items
+        for item in items :
+            try :
+                self.process_lease_item(lease, item)
+
+            except Exception as ex:
+                log.warn("Failed to process lease item: %s: %s:", lease_name, item, exc_info=True)
+        
+        # k
+        log.debug("%-15s: %s", lease_name, lease)
+
+        return lease
+
+    def log_lease (self, lease, old_lease=None) :
+        """
+            Log given lease transition on stdout.
+        """
+
+        # log
+        if old_lease :
+            log.info("%-15s: %20s @ %8s <- %-8s @ %20s", old_lease['lease'],
+                old_lease.get('ends', '???'),
+                old_lease.get('next-binding-state', ''),    # optional
+                old_lease.get('binding-state', '???'), 
+                old_lease.get('starts', '???'),
+            )
+
+        log.info("%-15s: %20s @ %8s -> %-8s @ %20s", lease['lease'], 
+                lease.get('starts', '???'),
+                lease.get('binding-state', '???'), 
+                lease.get('next-binding-state', ''),    # optional
+                lease.get('ends', '???'),
+        )
+
+    def process_block (self, blockdata, log_leases=False) :
+        """
+            Process given block (from DHCPLeasesParser.parse()), to update state.
+        """
+
+        block, items = blockdata
+
+        type = block.pop(0)
+        args = block
+
+        if type == 'lease' :
+            if len(args) != 1 :
+                return log.warn("lease block with weird args, ignore: %s", args)
+            
+            # the lease address
+            lease, = args
+
+            log.debug("lease: %s: %s", lease, items)
+            
+            if lease in self.leases :
+                old = self.leases[lease]
+            else :
+                old = None
+
+            new = self.process_lease(lease, items)
+
+            if log_leases :
+                self.log_lease(new, old)
+
+            return new
+
+        else :
+            log.warn("unknown block: %s: %s", type, args)
+
+    def process (self) :
+        """
+            Read new lines from the leases database and update our state.
+            
+            XXX: Returns
+                (sync, leases)
+
+            whereby sync is normally False, and leases the set of (possibly) changed leases, unless during initial
+            startup and on database replacement, when the sync is True, and the entire set of valid leases is returned.
+        """
+        
+        # handle file replace by reading until EOF
+        sync = False
+#        leases = []
+
+        if self.leases is None :
+            # initial sync
+            self.reset()
+            sync = True
+        
+        # parse in any new lines from TailFile... yields None if the file was replaced
+        for line in self.source.readlines(eof_mark=True) :
+            if line is None :
+                log.info("Reading new dhcpd.leases")
+
+                # resync
+                self.reset()
+                sync = True
+
+            else :
+                # parse
+                for blockdata in self.parser.parse(line) :
+                    # don't log if syncing, only on normal updates (next tail-cycle)
+                    lease = self.process_block(blockdata, log_leases=(not sync))
+
+                    #if not sync :
+                    #    leases.append(lease)
+                    yield lease
+
+#        if sync :
+#            return True, self.leases.values()
+#        else :
+#            return False, leases
+
+    def __iter__ (self) :
+        """
+            Iterate over all leases.
+        """
+
+        return self.leases.itervalues()
+
+    # utils
+    def lease_state (self, lease) :
+        """
+            Get state for lease.
+        """
+
+        # count by state
+        starts = lease.get('starts')
+        state = lease.get('binding-state', 'unknown')
+        next_state = lease.get('next-binding-state')
+        ends = lease.get('ends')
+
+        #log.debug("%-15s: %s: %8s -> %-8s: %s", ip, starts, state, next_state or '', ends)
+
+        if next_state and ends and ends < datetime.now() :
+            # XXX: mark as, "expired", even they next-binding-state is probably "free"
+            state = 'expired' # lease['next-binding-state']
+
+        return state
+
+if __name__ == '__main__' :
+    import logging
+
+    logging.basicConfig()
+
+    import doctest
+    doctest.testmod()
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pvl/dhcp/syslog.py	Sat Jan 26 11:09:30 2013 +0200
@@ -0,0 +1,194 @@
+"""
+    Parse ISC dhcpd messages in syslog.
+"""
+
+import re
+
+import logging; log = logging.getLogger('pvl.dhcp.syslog')
+
+# TODO: use pvl.syslog.rule.SyslogRule?
+class DHCPSyslogParser (object) :
+    """
+        Parse SyslogMessages from SyslogParser for ISC dhcp semantics.
+    """
+
+    ## various message types sent/recieved by dhcpd
+    # from server/dhcp.c
+    TYPE_NAMES = (
+        "DHCPDISCOVER",
+        "DHCPOFFER",
+        "DHCPREQUEST",
+        "DHCPDECLINE",
+        "DHCPACK",
+        "DHCPNAK",
+        "DHCPRELEASE",
+        "DHCPINFORM",
+        "type 9",
+        "DHCPLEASEQUERY",
+        "DHCPLEASEUNASSIGNED",
+        "DHCPLEASEUNKNOWN",
+        "DHCPLEASEACTIVE"
+    )
+
+    # message-parsing regexp..
+    RECV_MESSAGE_RE = (
+        # dhcpdiscover/ack_lease: info/error
+        #   hwaddr:     <no identifier>
+        #   hostname:   Hostname Unsuitable for Printing
+        #   error:
+        #               peer holds all free leases
+        #               network %s: no free leases
+        re.compile(r'(?P<type>DHCPDISCOVER) from (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?)(: (?P<error>.+?))?$'),
+
+        # dhcprequest
+        #   error:
+        #               wrong network.
+        #               ignored (not authoritative).
+        #               ignored (unknown subnet).
+        #               lease %s unavailable.
+        #               unknown lease %s.
+        re.compile(r'(?P<type>DHCPREQUEST) for (?P<lease>.+?)( \((?P<server>.+?)\))? from (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?)(: (?P<error>.+?))?$'),
+
+        # dhcprelease
+        re.compile(r'(?P<type>DHCPRELEASE) of (?P<lease>.+?) from (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?) \((?P<found>.+?)\)$'),
+
+        # dhcpdecline
+        #   status:
+        #       abandoned
+        #       not found
+        #       ignored
+        re.compile(r'(?P<type>DHCPDECLINE) of (?P<lease>.+?) from (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?): (?P<status>.+?)$'),
+
+        # dhcpinform
+        #   error:
+        #       ignored (null source address).
+        #       unknown subnet for relay address %s
+        #       unknown subnet for %s address %s
+        #       not authoritative for subnet %s
+        re.compile(r'(?P<type>DHCPINFORM) from (?P<lease>.+?) via (?P<gateway>.+?)(: (?P<error>.+?))?$'),
+        
+        # dhcpleasequery
+        re.compile(r'(?P<type>DHCPLEASEQUERY) from (?P<server>.+?)( for (?P<key_type>IP|client-id|MAC address) (?P<key>.+?))?(: (?P<error>.+?))?$'),
+
+        # dhcp: generic/unknown packet
+        re.compile(r'(?P<type>\w+) from (?P<hwaddr>.+?) via (?P<gateway>.+?): (?P<error>.+?)$'),
+    )
+
+    SEND_MESSAGE_RE = (
+        # dhcp_reply
+        re.compile(r'(?P<type>DHCPACK|DHCPOFFER|BOOTREPLY) on (?P<lease>.+?) to (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?)$'),
+
+        # dhcpinform
+        #   hwaddr:     <no client hardware address>
+        re.compile(r'(?P<type>DHCPACK) to (?P<lease>.+?) \((?P<hwaddr>.+?)\) via (?P<gateway>.+?)$'),
+
+        # nak_lease
+        re.compile(r'(?P<type>DHCPNAK) on (?P<lease>.+?) to (?P<hwaddr>.+?) via (?P<gateway>.+?)$'),
+
+        # dhcpleasequery
+        re.compile(r'(?P<type>DHCPLEASEUNKNOWN|DHCPLEASEACTIVE|DHCPLEASEUNASSIGNED) to (?P<lease>.+?) for (?P<key_type>IP|client-id|MAC address) (?P<key>.+?) \((?P<count>\d+) associated IPs\)$'),
+    )
+
+    MESSAGE_ERROR_RE = (
+        ('peer-all-free-leases',    re.compile('peer holds all free leases')),
+        ('no-free-leases',          re.compile(r'network (?P<network>.+?): no free leases')),
+        ('wrong-network',           re.compile(r'wrong network')),
+        ('ignored-not-auth',        re.compile(r'ignored \(not authoritative\)')),
+        ('ignored-unknown-subnet',  re.compile(r'ignored \(unknown subnet\)')),
+        ('lease-unavailable',       re.compile(r'lease (?P<lease>.+?) unavailable')),
+        ('lease-unknown',           re.compile(r'unknown lease (?P<lease>.+?).$')),
+    )
+
+    ERROR_RE = (
+        # find_lease
+        ('duplicate-uid-lease', 
+            re.compile(r'uid lease (?P<client>.+?) for client (?P<hwaddr>.+?) is duplicate on (?P<shared_network>.+?)$')),
+
+        # dhcprelease
+        ('dhcprelease-requested-address', 
+            re.compile(r'DHCPRELEASE from (?P<hwaddr>.+?) specified requested-address.')),
+
+        # ???
+        ('unexpected-icmp-echo-reply',
+            re.compile(r'unexpected ICMP Echo Reply from (?P<client>.+?)$')),
+        
+        ('host-unknown',
+            re.compile(r'(?P<host>.+?): host unknown.')),
+    )
+
+    IGNORE_RE = (
+        re.compile(r'Wrote (?P<count>\d+) (?P<what>.+?) to leases file.'),
+    )
+
+    def parse (self, line) :
+        """
+            Match line against our regexps, returning a
+
+                {
+                    tag:        send/recv/error,
+                    type:       ...,
+                    [error]:    ...,
+                    ...
+                }
+
+            dict if matched
+
+            Returns False if the message is ignored, or None if the no regexp matched.
+        """
+
+        for tag, re_list in (
+            ('recv',    self.RECV_MESSAGE_RE),
+            ('send',    self.SEND_MESSAGE_RE),
+        ) :
+            for re in re_list :
+                # test
+                match = re.match(line)
+
+                if match :
+                    data = match.groupdict()
+                    data['tag'] = tag
+
+                    return data
+                
+        # error?
+        for type, re in self.ERROR_RE:
+            match = re.match(line)
+
+            if match : 
+                data = match.groupdict()
+                data['tag'] = 'error'
+                data['type'] = type
+
+                return data
+
+        # ignore
+        for re in self.IGNORE_RE :
+            if re.match(line) :
+                # ignore
+                return False
+
+        # unknown
+        return None
+
+    def parse_error (self, error) :
+        """
+            Match given error status from send/recv against known types, returning a type name or None.
+        """
+
+        for type, re in self.MESSAGE_ERROR_RE :
+            match = re.match(error)
+
+            if match :
+                return type
+        
+        # nope
+        return None
+
+if __name__ == '__main__' :
+    import logging
+
+    logging.basicConfig()
+
+    import doctest
+    doctest.testmod()
+
--- a/pvl/syslog/dhcp.py	Fri Jan 25 22:02:55 2013 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,191 +0,0 @@
-"""
-    Parse ISC dhcpd messages in syslog.
-"""
-
-import re
-
-class DHCPSyslogFilter (object) :
-    """
-        Parse SyslogMessages from SyslogParser for ISC dhcp semantics.
-    """
-
-    ## various message types sent/recieved by dhcpd
-    # from server/dhcp.c
-    TYPE_NAMES = (
-        "DHCPDISCOVER",
-        "DHCPOFFER",
-        "DHCPREQUEST",
-        "DHCPDECLINE",
-        "DHCPACK",
-        "DHCPNAK",
-        "DHCPRELEASE",
-        "DHCPINFORM",
-        "type 9",
-        "DHCPLEASEQUERY",
-        "DHCPLEASEUNASSIGNED",
-        "DHCPLEASEUNKNOWN",
-        "DHCPLEASEACTIVE"
-    )
-
-    # message-parsing regexp..
-    RECV_MESSAGE_RE = (
-        # dhcpdiscover/ack_lease: info/error
-        #   hwaddr:     <no identifier>
-        #   hostname:   Hostname Unsuitable for Printing
-        #   error:
-        #               peer holds all free leases
-        #               network %s: no free leases
-        re.compile(r'(?P<type>DHCPDISCOVER) from (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?)(: (?P<error>.+?))?$'),
-
-        # dhcprequest
-        #   error:
-        #               wrong network.
-        #               ignored (not authoritative).
-        #               ignored (unknown subnet).
-        #               lease %s unavailable.
-        #               unknown lease %s.
-        re.compile(r'(?P<type>DHCPREQUEST) for (?P<lease>.+?)( \((?P<server>.+?)\))? from (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?)(: (?P<error>.+?))?$'),
-
-        # dhcprelease
-        re.compile(r'(?P<type>DHCPRELEASE) of (?P<lease>.+?) from (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?) \((?P<found>.+?)\)$'),
-
-        # dhcpdecline
-        #   status:
-        #       abandoned
-        #       not found
-        #       ignored
-        re.compile(r'(?P<type>DHCPDECLINE) of (?P<lease>.+?) from (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?): (?P<status>.+?)$'),
-
-        # dhcpinform
-        #   error:
-        #       ignored (null source address).
-        #       unknown subnet for relay address %s
-        #       unknown subnet for %s address %s
-        #       not authoritative for subnet %s
-        re.compile(r'(?P<type>DHCPINFORM) from (?P<lease>.+?) via (?P<gateway>.+?)(: (?P<error>.+?))?$'),
-        
-        # dhcpleasequery
-        re.compile(r'(?P<type>DHCPLEASEQUERY) from (?P<server>.+?)( for (?P<key_type>IP|client-id|MAC address) (?P<key>.+?))?(: (?P<error>.+?))?$'),
-
-        # dhcp: generic/unknown packet
-        re.compile(r'(?P<type>\w+) from (?P<hwaddr>.+?) via (?P<gateway>.+?): (?P<error>.+?)$'),
-    )
-
-    SEND_MESSAGE_RE = (
-        # dhcp_reply
-        re.compile(r'(?P<type>DHCPACK|DHCPOFFER|BOOTREPLY) on (?P<lease>.+?) to (?P<hwaddr>.+?)( \((?P<hostname>.+?)\))? via (?P<gateway>.+?)$'),
-
-        # dhcpinform
-        #   hwaddr:     <no client hardware address>
-        re.compile(r'(?P<type>DHCPACK) to (?P<lease>.+?) \((?P<hwaddr>.+?)\) via (?P<gateway>.+?)$'),
-
-        # nak_lease
-        re.compile(r'(?P<type>DHCPNAK) on (?P<lease>.+?) to (?P<hwaddr>.+?) via (?P<gateway>.+?)$'),
-
-        # dhcpleasequery
-        re.compile(r'(?P<type>DHCPLEASEUNKNOWN|DHCPLEASEACTIVE|DHCPLEASEUNASSIGNED) to (?P<lease>.+?) for (?P<key_type>IP|client-id|MAC address) (?P<key>.+?) \((?P<count>\d+) associated IPs\)$'),
-    )
-
-    MESSAGE_ERROR_RE = (
-        ('peer-all-free-leases',    re.compile('peer holds all free leases')),
-        ('no-free-leases',          re.compile(r'network (?P<network>.+?): no free leases')),
-        ('wrong-network',           re.compile(r'wrong network')),
-        ('ignored-not-auth',        re.compile(r'ignored \(not authoritative\)')),
-        ('ignored-unknown-subnet',  re.compile(r'ignored \(unknown subnet\)')),
-        ('lease-unavailable',       re.compile(r'lease (?P<lease>.+?) unavailable')),
-        ('lease-unknown',           re.compile(r'unknown lease (?P<lease>.+?).$')),
-    )
-
-    ERROR_RE = (
-        # find_lease
-        ('duplicate-uid-lease', 
-            re.compile(r'uid lease (?P<client>.+?) for client (?P<hwaddr>.+?) is duplicate on (?P<shared_network>.+?)$')),
-
-        # dhcprelease
-        ('dhcprelease-requested-address', 
-            re.compile(r'DHCPRELEASE from (?P<hwaddr>.+?) specified requested-address.')),
-
-        # ???
-        ('unexpected-icmp-echo-reply',
-            re.compile(r'unexpected ICMP Echo Reply from (?P<client>.+?)$')),
-        
-        ('host-unknown',
-            re.compile(r'(?P<host>.+?): host unknown.')),
-    )
-
-    IGNORE_RE = (
-        re.compile(r'Wrote (?P<count>\d+) (?P<what>.+?) to leases file.'),
-    )
-
-    def parse (self, line) :
-        """
-            Match line against our regexps, returning a
-
-                {
-                    tag:        send/recv/error,
-                    type:       ...,
-                    [error]:    ...,
-                    ...
-                }
-
-            dict if matched
-
-            Returns False if the message is ignored, or None if the no regexp matched.
-        """
-
-        for tag, re_list in (
-            ('recv',    self.RECV_MESSAGE_RE),
-            ('send',    self.SEND_MESSAGE_RE),
-        ) :
-            for re in re_list :
-                # test
-                match = re.match(line)
-
-                if match :
-                    data = match.groupdict()
-                    data['tag'] = tag
-
-                    return data
-                
-        # error?
-        for type, re in self.ERROR_RE:
-            match = re.match(line)
-
-            if match : 
-                data = match.groupdict()
-                data['tag'] = 'error'
-                data['type'] = type
-
-                return data
-
-        # ignore
-        for re in self.IGNORE_RE :
-            if re.match(line) :
-                # ignore
-                return False
-
-        # unknown
-        return None
-
-    def parse_error (self, error) :
-        """
-            Match given error status from send/recv against known types, returning a type name or None.
-        """
-
-        for type, re in self.MESSAGE_ERROR_RE :
-            match = re.match(error)
-
-            if match :
-                return type
-        
-        # nope
-        return None
-
-if __name__ == '__main__' :
-    import logging
-
-    logging.basicConfig()
-
-    import doctest
-    doctest.testmod()
-
--- a/pvl/verkko/dhcp/leases.py	Fri Jan 25 22:02:55 2013 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,450 +0,0 @@
-"""
-    DHCP dhcpd.leases handling/tracking
-"""
-
-import pvl.syslog.tail
-
-from datetime import datetime
-
-import logging; log = logging.getLogger('pvl.verkko.dhcp.leases')
-
-class DHCPLeasesParser (object) :
-    """
-        Simplistic parser for a dhcpd.leases file.
-
-        Doesn't implement the full spec, but a useful approximation.
-    """
-
-    def __init__ (self) :
-        self.block = None
-        self.items = []
-    
-    @classmethod
-    def split (cls, line) :
-        """
-            Split given line-data.
-            
-            >>> split = DHCPLeasesParser.split
-            >>> split('foo bar')
-            ['foo', 'bar']
-            >>> split('"foo"')
-            ['foo']
-            >>> split('foo "asdf quux" bar')
-            ['foo', 'asdf quux', 'bar']
-            >>> split('foo "asdf quux"')
-            ['foo', 'asdf quux']
-        """
-
-        # parse out one str
-        if '"' in line :
-            log.debug("%s", line)
-
-            # crude
-            pre, line = line.split('"', 1)
-            data, post = line.rsplit('"', 1)
-
-            return pre.split() + [data] + post.split()
-        else :
-            return line.split()
-
-    @classmethod
-    def lex (self, line) :
-        """
-            Yield tokens from the given lines.
-
-            >>> lex = DHCPLeasesParser.lex
-            >>> list(lex('foo;'))
-            [('item', ['foo'])]
-            >>> list(item for line in ['foo {', ' bar;', '}'] for item in lex(line))
-            [('open', ['foo']), ('item', ['bar']), ('close', None)]
-
-        """
-
-        log.debug("%s", line)
-
-        # comments?
-        if '#' in line :
-            line, comment = line.split('#', 1)
-        else :
-            comment = None
-
-        # clean?
-        line = line.strip()
-
-        # parse
-        if not line :
-            # ignore, empty/comment
-            return
-        
-        elif line.startswith('uid') :
-            # XXX: too hard to parse properly
-            return
-
-        elif '{' in line :
-            decl, line = line.split('{', 1)
-
-            # we are in a new decl
-            yield 'open', self.split(decl)
-       
-        elif ';' in line :
-            param, line = line.split(';', 1)
-            
-            # a stanza
-            yield 'item', self.split(param)
-        
-        elif '}' in line :
-            close, line = line.split('}', 1)
-
-            if close.strip() :
-                log.warn("Predata on close: %s", close)
-
-            # end
-            yield 'close', None
-    
-        else :
-            log.warn("Unknown line: %s", line)
-            return
-
-        # got the whole line?
-        if line.strip() :
-            log.warn("Data remains: %s", line)
-
-    def push_block (self, block) :
-        """
-            Open new block.
-        """
-        
-        # XXX: stack
-        assert not self.block
-
-        self.block = block
-        self.items = []
-
-    def feed_block (self, item) :
-        """
-            Add item to block
-        """
-
-        assert self.block
-
-        self.items.append(item)
-
-    def pop_block (self) :
-        """
-            Close block. Returns
-                (block, [items])
-        """
-
-        assert self.block
-
-        block, items = self.block, self.items
-
-        self.block = None
-        self.items = None
-
-        return block, items
-
-    def parse (self, line) :
-        """
-            Parse given line, yielding any complete blocks that come out.
-
-            >>> parser = DHCPLeasesParser()
-            >>> list(parser.parse_lines(['foo {', ' bar;', ' quux asdf;', '}']))
-            [(['foo'], [['bar'], ['quux', 'asdf']])]
-
-            >>> parser = DHCPLeasesParser()
-            >>> list(parser.parse('foo {'))
-            []
-            >>> list(parser.parse_lines([' bar;', ' quux asdf;']))
-            []
-            >>> list(parser.parse('}'))
-            [(['foo'], [['bar'], ['quux', 'asdf']])]
-        """
-
-        for token, args in self.lex(line) :
-            #log.debug("%s: %s [block=%s]", token, args, self.block)
-
-            if token == 'open' :
-                # open new block
-                block = args
-
-                if self.block :
-                    log.warn("nested blocks: %s > %s", self.block, block)
-                    continue
-            
-                log.debug("open block: %s", block)
-                self.push_block(block)
-            
-            elif token == 'close' :
-                log.debug("close block: %s", self.block)
-
-                # collected block items
-                yield self.pop_block()
-
-            # must be within block!
-            elif token == 'item' :
-                item = args
-
-                if not self.block :
-                    log.warn("token outside block: %s: %s", token, args)
-                    continue
-
-                log.debug("block %s item: %s", self.block, item)
-                self.feed_block(item)
-
-            else :
-                # ???
-                raise KeyError("Unknown token: {0}: {1}".format(token, args))
-    
-    def parse_lines (self, lines) :
-        """
-            Trivial wrapper around parse to parse multiple lines.
-        """
-
-        for line in lines :
-            for item in self.parse(line) :
-                yield item
-
-class DHCPLeasesDatabase (object) :
-    """
-        Process log-structured leases file.
-    """
-
-    LEASE_DATES = ('starts', 'ends', 'tstp', 'tsfp', 'atsfp', 'cltt')
-
-    # default format
-    LEASE_DATE_NEVER = 'never'
-    LEASE_DATE_FMT_DEFAULT = '%w %Y/%m/%d %H:%M:%S'
-
-    lease_date_fmt = LEASE_DATE_FMT_DEFAULT
-
-    def __init__ (self, path) :
-        """
-            path        - path to dhcpd.leases file
-        """
-
-        # tail; handles file re-writes
-        self.source = pvl.syslog.tail.TailFile(path)
-
-        # parser state
-        self.parser = DHCPLeasesParser()
-
-        # initial leases state
-        self.leases = None
-
-    def reset (self) :
-        """
-            Reset state, if we started to read a new file.
-        """
-
-        self.leases = {}
-
-    def process_lease_item_date (self, args) :
-        """
-            Process lease-item date spec into datetime.
-
-            Returns None if 'never'.
-        """
-
-        data = ' '.join(args)
-
-        if data == self.LEASE_DATE_NEVER :
-            return None
-        else :
-            return datetime.strptime(data, self.lease_date_fmt)
-
-    def process_lease_item (self, lease, item) :
-        """
-            Process a single item from the lease, updating the lease dict
-        """
-
-        item = list(item)
-
-        name = item.pop(0)
-        subname = item[0] if item else None
-
-        if name in self.LEASE_DATES :
-            lease[name] = self.process_lease_item_date(item)
-
-        elif name == 'hardware':
-            # args
-            lease['hwtype'], lease['hwaddr'] = item
-        
-        elif name == 'uid' :
-            lease['uid'], = item
-
-        elif name == 'client-hostname' :
-            lease['client-hostname'], = item
-        
-        elif name == 'abandoned' :
-            lease['abandoned'] = True
-
-        elif name == 'binding' :
-            _state, lease['binding-state'] = item
-
-        elif name == 'next' and subname == 'binding' :
-            _binding, _state, lease['next-binding-state'] = item
-
-        else :
-            log.warn("unknown lease item: %s: %s", name, item)
-
-    def process_lease (self, lease_name, items) :
-        """
-            Process given lease block to update our state.
-
-            Returns the lease object, and a possible old lease.
-        """
-
-        # replace any existing
-        lease = self.leases[lease_name] = {}
-
-        # meta
-        lease['lease'] = lease_name
-
-        # parse items
-        for item in items :
-            try :
-                self.process_lease_item(lease, item)
-
-            except Exception as ex:
-                log.warn("Failed to process lease item: %s: %s:", lease_name, item, exc_info=True)
-        
-        # k
-        log.debug("%-15s: %s", lease_name, lease)
-
-        return lease
-
-    def log_lease (self, lease, old_lease=None) :
-        """
-            Log given lease transition on stdout.
-        """
-
-        # log
-        if old_lease :
-            log.info("%-15s: %20s @ %8s <- %-8s @ %20s", old_lease['lease'],
-                old_lease.get('ends', '???'),
-                old_lease.get('next-binding-state', ''),    # optional
-                old_lease.get('binding-state', '???'), 
-                old_lease.get('starts', '???'),
-            )
-
-        log.info("%-15s: %20s @ %8s -> %-8s @ %20s", lease['lease'], 
-                lease.get('starts', '???'),
-                lease.get('binding-state', '???'), 
-                lease.get('next-binding-state', ''),    # optional
-                lease.get('ends', '???'),
-        )
-
-    def process_block (self, blockdata, log_leases=False) :
-        """
-            Process given block (from DHCPLeasesParser.parse()), to update state.
-        """
-
-        block, items = blockdata
-
-        type = block.pop(0)
-        args = block
-
-        if type == 'lease' :
-            if len(args) != 1 :
-                return log.warn("lease block with weird args, ignore: %s", args)
-            
-            # the lease address
-            lease, = args
-
-            log.debug("lease: %s: %s", lease, items)
-            
-            if lease in self.leases :
-                old = self.leases[lease]
-            else :
-                old = None
-
-            new = self.process_lease(lease, items)
-
-            if log_leases :
-                self.log_lease(new, old)
-
-            return new
-
-        else :
-            log.warn("unknown block: %s: %s", type, args)
-
-    def process (self) :
-        """
-            Read new lines from the leases database and update our state.
-            
-            XXX: Returns
-                (sync, leases)
-
-            whereby sync is normally False, and leases the set of (possibly) changed leases, unless during initial
-            startup and on database replacement, when the sync is True, and the entire set of valid leases is returned.
-        """
-        
-        # handle file replace by reading until EOF
-        sync = False
-#        leases = []
-
-        if self.leases is None :
-            # initial sync
-            self.reset()
-            sync = True
-        
-        # parse in any new lines from TailFile... yields None if the file was replaced
-        for line in self.source.readlines(eof_mark=True) :
-            if line is None :
-                log.info("Reading new dhcpd.leases")
-
-                # resync
-                self.reset()
-                sync = True
-
-            else :
-                # parse
-                for blockdata in self.parser.parse(line) :
-                    # don't log if syncing, only on normal updates (next tail-cycle)
-                    lease = self.process_block(blockdata, log_leases=(not sync))
-
-                    #if not sync :
-                    #    leases.append(lease)
-                    yield lease
-
-#        if sync :
-#            return True, self.leases.values()
-#        else :
-#            return False, leases
-
-    def __iter__ (self) :
-        """
-            Iterate over all leases.
-        """
-
-        return self.leases.itervalues()
-
-    # utils
-    def lease_state (self, lease) :
-        """
-            Get state for lease.
-        """
-
-        # count by state
-        starts = lease.get('starts')
-        state = lease.get('binding-state', 'unknown')
-        next_state = lease.get('next-binding-state')
-        ends = lease.get('ends')
-
-        #log.debug("%-15s: %s: %8s -> %-8s: %s", ip, starts, state, next_state or '', ends)
-
-        if next_state and ends and ends < datetime.now() :
-            # XXX: mark as, "expired", even they next-binding-state is probably "free"
-            state = 'expired' # lease['next-binding-state']
-
-        return state
-
-if __name__ == '__main__' :
-    import logging
-
-    logging.basicConfig()
-
-    import doctest
-    doctest.testmod()
-