# HG changeset patch # User Tero Marttila # Date 1359191370 -7200 # Node ID a81ca751664d2c88236d30147c7ba53332c3b9d1 # Parent 4e120851ff52c312f79eb13a4a15f966073e1799 move pvl.verkko.dhcp and pvl.syslog.dhcp to pvl.dhcp.leases/syslog diff -r 4e120851ff52 -r a81ca751664d bin/pvl.syslog-dhcp --- a/bin/pvl.syslog-dhcp Fri Jan 25 22:02:55 2013 +0200 +++ b/bin/pvl.syslog-dhcp Sat Jan 26 11:09:30 2013 +0200 @@ -10,8 +10,8 @@ import pvl.syslog.args import pvl.verkko.db as db -import pvl.syslog.dhcp -import pvl.verkko.dhcp.leases +import pvl.dhcp.syslog +import pvl.dhcp.leases import logging, optparse @@ -294,7 +294,7 @@ self.leasedb = DHCPLeasesDatabase(db) # XXX - self.filter = pvl.syslog.dhcp.DHCPSyslogFilter() + self.parser = pvl.dhcp.syslog.DHCPSyslogParser() def createdb (self) : """ @@ -309,7 +309,7 @@ Handle a single item read from syslog to DB. """ - dhcp_item = self.filter.parse(item['msg']) + dhcp_item = self.parser.parse(item['msg']) log.debug("%s: %s", item, dhcp_item) @@ -360,15 +360,10 @@ if self.leases : # process internally - #sync, leases = self.leases.process() leases = self.leases.process() - #if sync : - # self.sync_leases(leases) if leases : self.process_leases(leases) - #else : - # pass if not poll : # done @@ -408,7 +403,7 @@ # leases if options.leases_file : log.info("Open up DHCP leases...") - leases = pvl.verkko.dhcp.leases.DHCPLeasesDatabase(options.leases_file) + leases = pvl.dhcp.leases.DHCPLeasesDatabase(options.leases_file) # force polling interval if options.leases_tail : diff -r 4e120851ff52 -r a81ca751664d pvl/dhcp/leases.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pvl/dhcp/leases.py Sat Jan 26 11:09:30 2013 +0200 @@ -0,0 +1,450 @@ +""" + DHCP dhcpd.leases handling/tracking +""" + +import pvl.syslog.tail # TailFile + +from datetime import datetime + +import logging; log = logging.getLogger('pvl.dhcp.leases') + +class DHCPLeasesParser (object) : + """ + Simplistic parser for a dhcpd.leases file. + + Doesn't implement the full spec, but a useful approximation. + """ + + def __init__ (self) : + self.block = None + self.items = [] + + @classmethod + def split (cls, line) : + """ + Split given line-data. + + >>> split = DHCPLeasesParser.split + >>> split('foo bar') + ['foo', 'bar'] + >>> split('"foo"') + ['foo'] + >>> split('foo "asdf quux" bar') + ['foo', 'asdf quux', 'bar'] + >>> split('foo "asdf quux"') + ['foo', 'asdf quux'] + """ + + # parse out one str + if '"' in line : + log.debug("%s", line) + + # crude + pre, line = line.split('"', 1) + data, post = line.rsplit('"', 1) + + return pre.split() + [data] + post.split() + else : + return line.split() + + @classmethod + def lex (self, line) : + """ + Yield tokens from the given lines. + + >>> lex = DHCPLeasesParser.lex + >>> list(lex('foo;')) + [('item', ['foo'])] + >>> list(item for line in ['foo {', ' bar;', '}'] for item in lex(line)) + [('open', ['foo']), ('item', ['bar']), ('close', None)] + + """ + + log.debug("%s", line) + + # comments? + if '#' in line : + line, comment = line.split('#', 1) + else : + comment = None + + # clean? + line = line.strip() + + # parse + if not line : + # ignore, empty/comment + return + + elif line.startswith('uid') : + # XXX: too hard to parse properly + return + + elif '{' in line : + decl, line = line.split('{', 1) + + # we are in a new decl + yield 'open', self.split(decl) + + elif ';' in line : + param, line = line.split(';', 1) + + # a stanza + yield 'item', self.split(param) + + elif '}' in line : + close, line = line.split('}', 1) + + if close.strip() : + log.warn("Predata on close: %s", close) + + # end + yield 'close', None + + else : + log.warn("Unknown line: %s", line) + return + + # got the whole line? + if line.strip() : + log.warn("Data remains: %s", line) + + def push_block (self, block) : + """ + Open new block. + """ + + # XXX: stack + assert not self.block + + self.block = block + self.items = [] + + def feed_block (self, item) : + """ + Add item to block + """ + + assert self.block + + self.items.append(item) + + def pop_block (self) : + """ + Close block. Returns + (block, [items]) + """ + + assert self.block + + block, items = self.block, self.items + + self.block = None + self.items = None + + return block, items + + def parse (self, line) : + """ + Parse given line, yielding any complete blocks that come out. + + >>> parser = DHCPLeasesParser() + >>> list(parser.parse_lines(['foo {', ' bar;', ' quux asdf;', '}'])) + [(['foo'], [['bar'], ['quux', 'asdf']])] + + >>> parser = DHCPLeasesParser() + >>> list(parser.parse('foo {')) + [] + >>> list(parser.parse_lines([' bar;', ' quux asdf;'])) + [] + >>> list(parser.parse('}')) + [(['foo'], [['bar'], ['quux', 'asdf']])] + """ + + for token, args in self.lex(line) : + #log.debug("%s: %s [block=%s]", token, args, self.block) + + if token == 'open' : + # open new block + block = args + + if self.block : + log.warn("nested blocks: %s > %s", self.block, block) + continue + + log.debug("open block: %s", block) + self.push_block(block) + + elif token == 'close' : + log.debug("close block: %s", self.block) + + # collected block items + yield self.pop_block() + + # must be within block! + elif token == 'item' : + item = args + + if not self.block : + log.warn("token outside block: %s: %s", token, args) + continue + + log.debug("block %s item: %s", self.block, item) + self.feed_block(item) + + else : + # ??? + raise KeyError("Unknown token: {0}: {1}".format(token, args)) + + def parse_lines (self, lines) : + """ + Trivial wrapper around parse to parse multiple lines. + """ + + for line in lines : + for item in self.parse(line) : + yield item + +class DHCPLeasesDatabase (object) : + """ + Process log-structured leases file. + """ + + LEASE_DATES = ('starts', 'ends', 'tstp', 'tsfp', 'atsfp', 'cltt') + + # default format + LEASE_DATE_NEVER = 'never' + LEASE_DATE_FMT_DEFAULT = '%w %Y/%m/%d %H:%M:%S' + + lease_date_fmt = LEASE_DATE_FMT_DEFAULT + + def __init__ (self, path) : + """ + path - path to dhcpd.leases file + """ + + # tail; handles file re-writes + self.source = pvl.syslog.tail.TailFile(path) + + # parser state + self.parser = DHCPLeasesParser() + + # initial leases state + self.leases = None + + def reset (self) : + """ + Reset state, if we started to read a new file. + """ + + self.leases = {} + + def process_lease_item_date (self, args) : + """ + Process lease-item date spec into datetime. + + Returns None if 'never'. + """ + + data = ' '.join(args) + + if data == self.LEASE_DATE_NEVER : + return None + else : + return datetime.strptime(data, self.lease_date_fmt) + + def process_lease_item (self, lease, item) : + """ + Process a single item from the lease, updating the lease dict + """ + + item = list(item) + + name = item.pop(0) + subname = item[0] if item else None + + if name in self.LEASE_DATES : + lease[name] = self.process_lease_item_date(item) + + elif name == 'hardware': + # args + lease['hwtype'], lease['hwaddr'] = item + + elif name == 'uid' : + lease['uid'], = item + + elif name == 'client-hostname' : + lease['client-hostname'], = item + + elif name == 'abandoned' : + lease['abandoned'] = True + + elif name == 'binding' : + _state, lease['binding-state'] = item + + elif name == 'next' and subname == 'binding' : + _binding, _state, lease['next-binding-state'] = item + + else : + log.warn("unknown lease item: %s: %s", name, item) + + def process_lease (self, lease_name, items) : + """ + Process given lease block to update our state. + + Returns the lease object, and a possible old lease. + """ + + # replace any existing + lease = self.leases[lease_name] = {} + + # meta + lease['lease'] = lease_name + + # parse items + for item in items : + try : + self.process_lease_item(lease, item) + + except Exception as ex: + log.warn("Failed to process lease item: %s: %s:", lease_name, item, exc_info=True) + + # k + log.debug("%-15s: %s", lease_name, lease) + + return lease + + def log_lease (self, lease, old_lease=None) : + """ + Log given lease transition on stdout. + """ + + # log + if old_lease : + log.info("%-15s: %20s @ %8s <- %-8s @ %20s", old_lease['lease'], + old_lease.get('ends', '???'), + old_lease.get('next-binding-state', ''), # optional + old_lease.get('binding-state', '???'), + old_lease.get('starts', '???'), + ) + + log.info("%-15s: %20s @ %8s -> %-8s @ %20s", lease['lease'], + lease.get('starts', '???'), + lease.get('binding-state', '???'), + lease.get('next-binding-state', ''), # optional + lease.get('ends', '???'), + ) + + def process_block (self, blockdata, log_leases=False) : + """ + Process given block (from DHCPLeasesParser.parse()), to update state. + """ + + block, items = blockdata + + type = block.pop(0) + args = block + + if type == 'lease' : + if len(args) != 1 : + return log.warn("lease block with weird args, ignore: %s", args) + + # the lease address + lease, = args + + log.debug("lease: %s: %s", lease, items) + + if lease in self.leases : + old = self.leases[lease] + else : + old = None + + new = self.process_lease(lease, items) + + if log_leases : + self.log_lease(new, old) + + return new + + else : + log.warn("unknown block: %s: %s", type, args) + + def process (self) : + """ + Read new lines from the leases database and update our state. + + XXX: Returns + (sync, leases) + + whereby sync is normally False, and leases the set of (possibly) changed leases, unless during initial + startup and on database replacement, when the sync is True, and the entire set of valid leases is returned. + """ + + # handle file replace by reading until EOF + sync = False +# leases = [] + + if self.leases is None : + # initial sync + self.reset() + sync = True + + # parse in any new lines from TailFile... yields None if the file was replaced + for line in self.source.readlines(eof_mark=True) : + if line is None : + log.info("Reading new dhcpd.leases") + + # resync + self.reset() + sync = True + + else : + # parse + for blockdata in self.parser.parse(line) : + # don't log if syncing, only on normal updates (next tail-cycle) + lease = self.process_block(blockdata, log_leases=(not sync)) + + #if not sync : + # leases.append(lease) + yield lease + +# if sync : +# return True, self.leases.values() +# else : +# return False, leases + + def __iter__ (self) : + """ + Iterate over all leases. + """ + + return self.leases.itervalues() + + # utils + def lease_state (self, lease) : + """ + Get state for lease. + """ + + # count by state + starts = lease.get('starts') + state = lease.get('binding-state', 'unknown') + next_state = lease.get('next-binding-state') + ends = lease.get('ends') + + #log.debug("%-15s: %s: %8s -> %-8s: %s", ip, starts, state, next_state or '', ends) + + if next_state and ends and ends < datetime.now() : + # XXX: mark as, "expired", even they next-binding-state is probably "free" + state = 'expired' # lease['next-binding-state'] + + return state + +if __name__ == '__main__' : + import logging + + logging.basicConfig() + + import doctest + doctest.testmod() + diff -r 4e120851ff52 -r a81ca751664d pvl/dhcp/syslog.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pvl/dhcp/syslog.py Sat Jan 26 11:09:30 2013 +0200 @@ -0,0 +1,194 @@ +""" + Parse ISC dhcpd messages in syslog. +""" + +import re + +import logging; log = logging.getLogger('pvl.dhcp.syslog') + +# TODO: use pvl.syslog.rule.SyslogRule? +class DHCPSyslogParser (object) : + """ + Parse SyslogMessages from SyslogParser for ISC dhcp semantics. + """ + + ## various message types sent/recieved by dhcpd + # from server/dhcp.c + TYPE_NAMES = ( + "DHCPDISCOVER", + "DHCPOFFER", + "DHCPREQUEST", + "DHCPDECLINE", + "DHCPACK", + "DHCPNAK", + "DHCPRELEASE", + "DHCPINFORM", + "type 9", + "DHCPLEASEQUERY", + "DHCPLEASEUNASSIGNED", + "DHCPLEASEUNKNOWN", + "DHCPLEASEACTIVE" + ) + + # message-parsing regexp.. + RECV_MESSAGE_RE = ( + # dhcpdiscover/ack_lease: info/error + # hwaddr: + # hostname: Hostname Unsuitable for Printing + # error: + # peer holds all free leases + # network %s: no free leases + re.compile(r'(?PDHCPDISCOVER) from (?P.+?)( \((?P.+?)\))? via (?P.+?)(: (?P.+?))?$'), + + # dhcprequest + # error: + # wrong network. + # ignored (not authoritative). + # ignored (unknown subnet). + # lease %s unavailable. + # unknown lease %s. + re.compile(r'(?PDHCPREQUEST) for (?P.+?)( \((?P.+?)\))? from (?P.+?)( \((?P.+?)\))? via (?P.+?)(: (?P.+?))?$'), + + # dhcprelease + re.compile(r'(?PDHCPRELEASE) of (?P.+?) from (?P.+?)( \((?P.+?)\))? via (?P.+?) \((?P.+?)\)$'), + + # dhcpdecline + # status: + # abandoned + # not found + # ignored + re.compile(r'(?PDHCPDECLINE) of (?P.+?) from (?P.+?)( \((?P.+?)\))? via (?P.+?): (?P.+?)$'), + + # dhcpinform + # error: + # ignored (null source address). + # unknown subnet for relay address %s + # unknown subnet for %s address %s + # not authoritative for subnet %s + re.compile(r'(?PDHCPINFORM) from (?P.+?) via (?P.+?)(: (?P.+?))?$'), + + # dhcpleasequery + re.compile(r'(?PDHCPLEASEQUERY) from (?P.+?)( for (?PIP|client-id|MAC address) (?P.+?))?(: (?P.+?))?$'), + + # dhcp: generic/unknown packet + re.compile(r'(?P\w+) from (?P.+?) via (?P.+?): (?P.+?)$'), + ) + + SEND_MESSAGE_RE = ( + # dhcp_reply + re.compile(r'(?PDHCPACK|DHCPOFFER|BOOTREPLY) on (?P.+?) to (?P.+?)( \((?P.+?)\))? via (?P.+?)$'), + + # dhcpinform + # hwaddr: + re.compile(r'(?PDHCPACK) to (?P.+?) \((?P.+?)\) via (?P.+?)$'), + + # nak_lease + re.compile(r'(?PDHCPNAK) on (?P.+?) to (?P.+?) via (?P.+?)$'), + + # dhcpleasequery + re.compile(r'(?PDHCPLEASEUNKNOWN|DHCPLEASEACTIVE|DHCPLEASEUNASSIGNED) to (?P.+?) for (?PIP|client-id|MAC address) (?P.+?) \((?P\d+) associated IPs\)$'), + ) + + MESSAGE_ERROR_RE = ( + ('peer-all-free-leases', re.compile('peer holds all free leases')), + ('no-free-leases', re.compile(r'network (?P.+?): no free leases')), + ('wrong-network', re.compile(r'wrong network')), + ('ignored-not-auth', re.compile(r'ignored \(not authoritative\)')), + ('ignored-unknown-subnet', re.compile(r'ignored \(unknown subnet\)')), + ('lease-unavailable', re.compile(r'lease (?P.+?) unavailable')), + ('lease-unknown', re.compile(r'unknown lease (?P.+?).$')), + ) + + ERROR_RE = ( + # find_lease + ('duplicate-uid-lease', + re.compile(r'uid lease (?P.+?) for client (?P.+?) is duplicate on (?P.+?)$')), + + # dhcprelease + ('dhcprelease-requested-address', + re.compile(r'DHCPRELEASE from (?P.+?) specified requested-address.')), + + # ??? + ('unexpected-icmp-echo-reply', + re.compile(r'unexpected ICMP Echo Reply from (?P.+?)$')), + + ('host-unknown', + re.compile(r'(?P.+?): host unknown.')), + ) + + IGNORE_RE = ( + re.compile(r'Wrote (?P\d+) (?P.+?) to leases file.'), + ) + + def parse (self, line) : + """ + Match line against our regexps, returning a + + { + tag: send/recv/error, + type: ..., + [error]: ..., + ... + } + + dict if matched + + Returns False if the message is ignored, or None if the no regexp matched. + """ + + for tag, re_list in ( + ('recv', self.RECV_MESSAGE_RE), + ('send', self.SEND_MESSAGE_RE), + ) : + for re in re_list : + # test + match = re.match(line) + + if match : + data = match.groupdict() + data['tag'] = tag + + return data + + # error? + for type, re in self.ERROR_RE: + match = re.match(line) + + if match : + data = match.groupdict() + data['tag'] = 'error' + data['type'] = type + + return data + + # ignore + for re in self.IGNORE_RE : + if re.match(line) : + # ignore + return False + + # unknown + return None + + def parse_error (self, error) : + """ + Match given error status from send/recv against known types, returning a type name or None. + """ + + for type, re in self.MESSAGE_ERROR_RE : + match = re.match(error) + + if match : + return type + + # nope + return None + +if __name__ == '__main__' : + import logging + + logging.basicConfig() + + import doctest + doctest.testmod() + diff -r 4e120851ff52 -r a81ca751664d pvl/syslog/dhcp.py --- a/pvl/syslog/dhcp.py Fri Jan 25 22:02:55 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,191 +0,0 @@ -""" - Parse ISC dhcpd messages in syslog. -""" - -import re - -class DHCPSyslogFilter (object) : - """ - Parse SyslogMessages from SyslogParser for ISC dhcp semantics. - """ - - ## various message types sent/recieved by dhcpd - # from server/dhcp.c - TYPE_NAMES = ( - "DHCPDISCOVER", - "DHCPOFFER", - "DHCPREQUEST", - "DHCPDECLINE", - "DHCPACK", - "DHCPNAK", - "DHCPRELEASE", - "DHCPINFORM", - "type 9", - "DHCPLEASEQUERY", - "DHCPLEASEUNASSIGNED", - "DHCPLEASEUNKNOWN", - "DHCPLEASEACTIVE" - ) - - # message-parsing regexp.. - RECV_MESSAGE_RE = ( - # dhcpdiscover/ack_lease: info/error - # hwaddr: - # hostname: Hostname Unsuitable for Printing - # error: - # peer holds all free leases - # network %s: no free leases - re.compile(r'(?PDHCPDISCOVER) from (?P.+?)( \((?P.+?)\))? via (?P.+?)(: (?P.+?))?$'), - - # dhcprequest - # error: - # wrong network. - # ignored (not authoritative). - # ignored (unknown subnet). - # lease %s unavailable. - # unknown lease %s. - re.compile(r'(?PDHCPREQUEST) for (?P.+?)( \((?P.+?)\))? from (?P.+?)( \((?P.+?)\))? via (?P.+?)(: (?P.+?))?$'), - - # dhcprelease - re.compile(r'(?PDHCPRELEASE) of (?P.+?) from (?P.+?)( \((?P.+?)\))? via (?P.+?) \((?P.+?)\)$'), - - # dhcpdecline - # status: - # abandoned - # not found - # ignored - re.compile(r'(?PDHCPDECLINE) of (?P.+?) from (?P.+?)( \((?P.+?)\))? via (?P.+?): (?P.+?)$'), - - # dhcpinform - # error: - # ignored (null source address). - # unknown subnet for relay address %s - # unknown subnet for %s address %s - # not authoritative for subnet %s - re.compile(r'(?PDHCPINFORM) from (?P.+?) via (?P.+?)(: (?P.+?))?$'), - - # dhcpleasequery - re.compile(r'(?PDHCPLEASEQUERY) from (?P.+?)( for (?PIP|client-id|MAC address) (?P.+?))?(: (?P.+?))?$'), - - # dhcp: generic/unknown packet - re.compile(r'(?P\w+) from (?P.+?) via (?P.+?): (?P.+?)$'), - ) - - SEND_MESSAGE_RE = ( - # dhcp_reply - re.compile(r'(?PDHCPACK|DHCPOFFER|BOOTREPLY) on (?P.+?) to (?P.+?)( \((?P.+?)\))? via (?P.+?)$'), - - # dhcpinform - # hwaddr: - re.compile(r'(?PDHCPACK) to (?P.+?) \((?P.+?)\) via (?P.+?)$'), - - # nak_lease - re.compile(r'(?PDHCPNAK) on (?P.+?) to (?P.+?) via (?P.+?)$'), - - # dhcpleasequery - re.compile(r'(?PDHCPLEASEUNKNOWN|DHCPLEASEACTIVE|DHCPLEASEUNASSIGNED) to (?P.+?) for (?PIP|client-id|MAC address) (?P.+?) \((?P\d+) associated IPs\)$'), - ) - - MESSAGE_ERROR_RE = ( - ('peer-all-free-leases', re.compile('peer holds all free leases')), - ('no-free-leases', re.compile(r'network (?P.+?): no free leases')), - ('wrong-network', re.compile(r'wrong network')), - ('ignored-not-auth', re.compile(r'ignored \(not authoritative\)')), - ('ignored-unknown-subnet', re.compile(r'ignored \(unknown subnet\)')), - ('lease-unavailable', re.compile(r'lease (?P.+?) unavailable')), - ('lease-unknown', re.compile(r'unknown lease (?P.+?).$')), - ) - - ERROR_RE = ( - # find_lease - ('duplicate-uid-lease', - re.compile(r'uid lease (?P.+?) for client (?P.+?) is duplicate on (?P.+?)$')), - - # dhcprelease - ('dhcprelease-requested-address', - re.compile(r'DHCPRELEASE from (?P.+?) specified requested-address.')), - - # ??? - ('unexpected-icmp-echo-reply', - re.compile(r'unexpected ICMP Echo Reply from (?P.+?)$')), - - ('host-unknown', - re.compile(r'(?P.+?): host unknown.')), - ) - - IGNORE_RE = ( - re.compile(r'Wrote (?P\d+) (?P.+?) to leases file.'), - ) - - def parse (self, line) : - """ - Match line against our regexps, returning a - - { - tag: send/recv/error, - type: ..., - [error]: ..., - ... - } - - dict if matched - - Returns False if the message is ignored, or None if the no regexp matched. - """ - - for tag, re_list in ( - ('recv', self.RECV_MESSAGE_RE), - ('send', self.SEND_MESSAGE_RE), - ) : - for re in re_list : - # test - match = re.match(line) - - if match : - data = match.groupdict() - data['tag'] = tag - - return data - - # error? - for type, re in self.ERROR_RE: - match = re.match(line) - - if match : - data = match.groupdict() - data['tag'] = 'error' - data['type'] = type - - return data - - # ignore - for re in self.IGNORE_RE : - if re.match(line) : - # ignore - return False - - # unknown - return None - - def parse_error (self, error) : - """ - Match given error status from send/recv against known types, returning a type name or None. - """ - - for type, re in self.MESSAGE_ERROR_RE : - match = re.match(error) - - if match : - return type - - # nope - return None - -if __name__ == '__main__' : - import logging - - logging.basicConfig() - - import doctest - doctest.testmod() - diff -r 4e120851ff52 -r a81ca751664d pvl/verkko/dhcp/leases.py --- a/pvl/verkko/dhcp/leases.py Fri Jan 25 22:02:55 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,450 +0,0 @@ -""" - DHCP dhcpd.leases handling/tracking -""" - -import pvl.syslog.tail - -from datetime import datetime - -import logging; log = logging.getLogger('pvl.verkko.dhcp.leases') - -class DHCPLeasesParser (object) : - """ - Simplistic parser for a dhcpd.leases file. - - Doesn't implement the full spec, but a useful approximation. - """ - - def __init__ (self) : - self.block = None - self.items = [] - - @classmethod - def split (cls, line) : - """ - Split given line-data. - - >>> split = DHCPLeasesParser.split - >>> split('foo bar') - ['foo', 'bar'] - >>> split('"foo"') - ['foo'] - >>> split('foo "asdf quux" bar') - ['foo', 'asdf quux', 'bar'] - >>> split('foo "asdf quux"') - ['foo', 'asdf quux'] - """ - - # parse out one str - if '"' in line : - log.debug("%s", line) - - # crude - pre, line = line.split('"', 1) - data, post = line.rsplit('"', 1) - - return pre.split() + [data] + post.split() - else : - return line.split() - - @classmethod - def lex (self, line) : - """ - Yield tokens from the given lines. - - >>> lex = DHCPLeasesParser.lex - >>> list(lex('foo;')) - [('item', ['foo'])] - >>> list(item for line in ['foo {', ' bar;', '}'] for item in lex(line)) - [('open', ['foo']), ('item', ['bar']), ('close', None)] - - """ - - log.debug("%s", line) - - # comments? - if '#' in line : - line, comment = line.split('#', 1) - else : - comment = None - - # clean? - line = line.strip() - - # parse - if not line : - # ignore, empty/comment - return - - elif line.startswith('uid') : - # XXX: too hard to parse properly - return - - elif '{' in line : - decl, line = line.split('{', 1) - - # we are in a new decl - yield 'open', self.split(decl) - - elif ';' in line : - param, line = line.split(';', 1) - - # a stanza - yield 'item', self.split(param) - - elif '}' in line : - close, line = line.split('}', 1) - - if close.strip() : - log.warn("Predata on close: %s", close) - - # end - yield 'close', None - - else : - log.warn("Unknown line: %s", line) - return - - # got the whole line? - if line.strip() : - log.warn("Data remains: %s", line) - - def push_block (self, block) : - """ - Open new block. - """ - - # XXX: stack - assert not self.block - - self.block = block - self.items = [] - - def feed_block (self, item) : - """ - Add item to block - """ - - assert self.block - - self.items.append(item) - - def pop_block (self) : - """ - Close block. Returns - (block, [items]) - """ - - assert self.block - - block, items = self.block, self.items - - self.block = None - self.items = None - - return block, items - - def parse (self, line) : - """ - Parse given line, yielding any complete blocks that come out. - - >>> parser = DHCPLeasesParser() - >>> list(parser.parse_lines(['foo {', ' bar;', ' quux asdf;', '}'])) - [(['foo'], [['bar'], ['quux', 'asdf']])] - - >>> parser = DHCPLeasesParser() - >>> list(parser.parse('foo {')) - [] - >>> list(parser.parse_lines([' bar;', ' quux asdf;'])) - [] - >>> list(parser.parse('}')) - [(['foo'], [['bar'], ['quux', 'asdf']])] - """ - - for token, args in self.lex(line) : - #log.debug("%s: %s [block=%s]", token, args, self.block) - - if token == 'open' : - # open new block - block = args - - if self.block : - log.warn("nested blocks: %s > %s", self.block, block) - continue - - log.debug("open block: %s", block) - self.push_block(block) - - elif token == 'close' : - log.debug("close block: %s", self.block) - - # collected block items - yield self.pop_block() - - # must be within block! - elif token == 'item' : - item = args - - if not self.block : - log.warn("token outside block: %s: %s", token, args) - continue - - log.debug("block %s item: %s", self.block, item) - self.feed_block(item) - - else : - # ??? - raise KeyError("Unknown token: {0}: {1}".format(token, args)) - - def parse_lines (self, lines) : - """ - Trivial wrapper around parse to parse multiple lines. - """ - - for line in lines : - for item in self.parse(line) : - yield item - -class DHCPLeasesDatabase (object) : - """ - Process log-structured leases file. - """ - - LEASE_DATES = ('starts', 'ends', 'tstp', 'tsfp', 'atsfp', 'cltt') - - # default format - LEASE_DATE_NEVER = 'never' - LEASE_DATE_FMT_DEFAULT = '%w %Y/%m/%d %H:%M:%S' - - lease_date_fmt = LEASE_DATE_FMT_DEFAULT - - def __init__ (self, path) : - """ - path - path to dhcpd.leases file - """ - - # tail; handles file re-writes - self.source = pvl.syslog.tail.TailFile(path) - - # parser state - self.parser = DHCPLeasesParser() - - # initial leases state - self.leases = None - - def reset (self) : - """ - Reset state, if we started to read a new file. - """ - - self.leases = {} - - def process_lease_item_date (self, args) : - """ - Process lease-item date spec into datetime. - - Returns None if 'never'. - """ - - data = ' '.join(args) - - if data == self.LEASE_DATE_NEVER : - return None - else : - return datetime.strptime(data, self.lease_date_fmt) - - def process_lease_item (self, lease, item) : - """ - Process a single item from the lease, updating the lease dict - """ - - item = list(item) - - name = item.pop(0) - subname = item[0] if item else None - - if name in self.LEASE_DATES : - lease[name] = self.process_lease_item_date(item) - - elif name == 'hardware': - # args - lease['hwtype'], lease['hwaddr'] = item - - elif name == 'uid' : - lease['uid'], = item - - elif name == 'client-hostname' : - lease['client-hostname'], = item - - elif name == 'abandoned' : - lease['abandoned'] = True - - elif name == 'binding' : - _state, lease['binding-state'] = item - - elif name == 'next' and subname == 'binding' : - _binding, _state, lease['next-binding-state'] = item - - else : - log.warn("unknown lease item: %s: %s", name, item) - - def process_lease (self, lease_name, items) : - """ - Process given lease block to update our state. - - Returns the lease object, and a possible old lease. - """ - - # replace any existing - lease = self.leases[lease_name] = {} - - # meta - lease['lease'] = lease_name - - # parse items - for item in items : - try : - self.process_lease_item(lease, item) - - except Exception as ex: - log.warn("Failed to process lease item: %s: %s:", lease_name, item, exc_info=True) - - # k - log.debug("%-15s: %s", lease_name, lease) - - return lease - - def log_lease (self, lease, old_lease=None) : - """ - Log given lease transition on stdout. - """ - - # log - if old_lease : - log.info("%-15s: %20s @ %8s <- %-8s @ %20s", old_lease['lease'], - old_lease.get('ends', '???'), - old_lease.get('next-binding-state', ''), # optional - old_lease.get('binding-state', '???'), - old_lease.get('starts', '???'), - ) - - log.info("%-15s: %20s @ %8s -> %-8s @ %20s", lease['lease'], - lease.get('starts', '???'), - lease.get('binding-state', '???'), - lease.get('next-binding-state', ''), # optional - lease.get('ends', '???'), - ) - - def process_block (self, blockdata, log_leases=False) : - """ - Process given block (from DHCPLeasesParser.parse()), to update state. - """ - - block, items = blockdata - - type = block.pop(0) - args = block - - if type == 'lease' : - if len(args) != 1 : - return log.warn("lease block with weird args, ignore: %s", args) - - # the lease address - lease, = args - - log.debug("lease: %s: %s", lease, items) - - if lease in self.leases : - old = self.leases[lease] - else : - old = None - - new = self.process_lease(lease, items) - - if log_leases : - self.log_lease(new, old) - - return new - - else : - log.warn("unknown block: %s: %s", type, args) - - def process (self) : - """ - Read new lines from the leases database and update our state. - - XXX: Returns - (sync, leases) - - whereby sync is normally False, and leases the set of (possibly) changed leases, unless during initial - startup and on database replacement, when the sync is True, and the entire set of valid leases is returned. - """ - - # handle file replace by reading until EOF - sync = False -# leases = [] - - if self.leases is None : - # initial sync - self.reset() - sync = True - - # parse in any new lines from TailFile... yields None if the file was replaced - for line in self.source.readlines(eof_mark=True) : - if line is None : - log.info("Reading new dhcpd.leases") - - # resync - self.reset() - sync = True - - else : - # parse - for blockdata in self.parser.parse(line) : - # don't log if syncing, only on normal updates (next tail-cycle) - lease = self.process_block(blockdata, log_leases=(not sync)) - - #if not sync : - # leases.append(lease) - yield lease - -# if sync : -# return True, self.leases.values() -# else : -# return False, leases - - def __iter__ (self) : - """ - Iterate over all leases. - """ - - return self.leases.itervalues() - - # utils - def lease_state (self, lease) : - """ - Get state for lease. - """ - - # count by state - starts = lease.get('starts') - state = lease.get('binding-state', 'unknown') - next_state = lease.get('next-binding-state') - ends = lease.get('ends') - - #log.debug("%-15s: %s: %8s -> %-8s: %s", ip, starts, state, next_state or '', ends) - - if next_state and ends and ends < datetime.now() : - # XXX: mark as, "expired", even they next-binding-state is probably "free" - state = 'expired' # lease['next-binding-state'] - - return state - -if __name__ == '__main__' : - import logging - - logging.basicConfig() - - import doctest - doctest.testmod() -