# HG changeset patch # User Tero Marttila # Date 1424792136 -7200 # Node ID 1d755df7bf97029ac1490fbfcdf81d3e5eeb3508 # Parent 6a8ea0d363c15d2a288b3c444aee5fc37093e383 pvl.hosts: refactor as a package; cleanup pvl.hosts.config with some basic tests diff -r 6a8ea0d363c1 -r 1d755df7bf97 pvl/hosts.py --- a/pvl/hosts.py Tue Feb 24 14:53:50 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,404 +0,0 @@ -""" - Host definitions. -""" - -import pvl.args -import pvl.dns.zone - -import collections -import configobj -import ipaddr -import optparse -import os.path - -import logging; log = logging.getLogger('pvl.hosts') - -__version__ = '0.8.0-dev' - -def optparser (parser) : - hosts = optparse.OptionGroup(parser, "Hosts input") - hosts.add_option('--hosts-charset', metavar='CHARSET', default='utf-8', - help="Encoding used for host files") - - hosts.add_option('--hosts-domain', metavar='DOMAIN', - help="Default domain for hosts") - - hosts.add_option('--hosts-include', metavar='PATH', - help="Optional path for hosts includes, beyond host config dir") - - return hosts - -class HostError (Exception) : - pass - -class HostConfigError (HostError) : - pass - -def parse_ethernet (value) : - """ - Normalize ethernet str. - """ - - return ':'.join('%02x' % int(x, 16) for x in value.split(':')) - -class Host (object) : - # the label used for alias4/6 hosts - ALIAS4_FMT = '{host}-ipv4' - ALIAS6_FMT = '{host}-ipv6' - - @classmethod - def expand (cls, options, range, host, domain, ip, forward=None, reverse=None, **opts) : - host = pvl.dns.zone.parse_generate_field(host) - ip = pvl.dns.zone.parse_generate_field(ip) - - if forward is not None : - _forward = pvl.dns.zone.parse_generate_field(forward) - else : - _forward = lambda i: forward - - if reverse is not None : - _reverse = pvl.dns.zone.parse_generate_field(reverse) - else : - _reverse = lambda i: reverse - - for i in range : - yield cls.build(options, host(i), domain, - ip = ip(i), - forward = _forward(i), - reverse = _reverse(i), - **opts - ) - - @classmethod - def config (cls, options, host, section=None, domain=None, ip=None, **extra) : - """ - Yield Hosts from a config section's scalars. - - options - pvl.args Options - host - the name of the section (or file) containing this host item. - section - the parent section containing this host item, or None - used for domain - """ - - if domain : - log.debug("%s: explicit domain: %s", host, domain) - elif '.' in host : - log.debug("%s: using as fqdn without domain", host) - domain = None - elif options.hosts_domain : - log.debug("%s: default domain to --hots-domain: %s", host, options.hosts_domain) - domain = options.hosts_domain - elif section : - log.debug("%s: default domain to section: %s", host, section) - domain = section - else : - raise ValueError("%s: no domain given" % (host, )) - - if '{' in host : - pre, host = host.split('{', 1) - range, post = host.rsplit('}', 1) - - range = pvl.dns.zone.parse_generate_range(range) - host = pre + "$" + post - - for host in cls.expand(options, range, host, domain, ip, **extra) : - yield host - else : - yield cls.build(options, host, domain, ip=ip, **extra) - - @classmethod - def build (cls, options, host, domain, - ip=None, ip6=None, owner=None, location=None, - alias=None, alias4=None, alias6=None, - forward=None, reverse=None, - down=None, - boot=None, - **extra) : - """ - Return a Host from a config section's scalars. - - """ - - if alias : - alias = alias.split() - else : - alias = () - - if alias4 : - alias4 = alias4.split() - else : - alias4 = () - - if alias6 : - alias6 = alias6.split() - else : - alias6 = () - - ethernet = { } - extensions = collections.defaultdict(dict) - - for field, value in extra.iteritems() : - if ':' in field : - extension, field = field.split(':', 1) - - extensions[extension][field] = value - - continue - - if '.' in field : - field, instance = field.split('.') - else : - instance = None - - if field == 'ethernet' : - if instance : - ethernet[instance] = parse_ethernet(value) - else : - for eth in value.split() : - ethernet[len(ethernet)] = parse_ethernet(eth) - else : - raise ValueError("%s: Unknown host field: %s=%s" % (host, field, value)) - - - if forward is None : - # normal zone - pass - elif forward : - # alias to external zone - pass - else : - # omit - forward = False - - if reverse is None : - # normal zone - pass - elif reverse : - # alias to external zone - pass - else : - # omit - reverse = False - - if down : - down = True - else : - down = None - - if not location : - location = location_domain = None - elif '@' in location: - location, location_domain = location.split('@', 1) - else: - location_domain = None - - return cls(host, - domain = domain, - ip = ipaddr.IPv4Address(ip) if ip else None, - ip6 = ipaddr.IPv6Address(ip6) if ip6 else None, - ethernet = ethernet, - alias = alias, - alias4 = alias4, - alias6 = alias6, - owner = owner, - location = location, - location_domain = location_domain, - boot = boot, - forward = forward, - reverse = reverse, - down = down, - - **extensions - ) - - def __init__ (self, host, - domain=None, - ip=None, ip6=None, - ethernet={ }, - alias=(), - owner=None, - location=None, - location_domain=None, - boot=None, - alias4=None, alias6=None, - forward=None, reverse=None, - down=None, - **extensions - ) : - """ - host - str - domain - str - ip - ipaddr.IPv4Address - ip6 - ipaddr.IPv6Address - ethernet - { index: ethernet } - alias - list - owner - str: LDAP uid - location - location name@ part - location_domain - location @domain part - alias4 - list (CNAME -> A) - alias6 - list (CNAME -> AAAA) - forward - generate forward records, or CNAME into given zone - reverse - generate reverse records, or CNAME into given zone - down - not online - """ - - self.host = host - self.domain = domain - self.ip = ip - self.ip6 = ip6 - self.ethernet = ethernet - self.alias = alias - self.alias4 = alias4 - self.alias6 = alias6 - self.owner = owner - self.location = location - self.location_domain = location_domain - self.boot = boot - self.forward = forward - self.reverse = reverse - self.down = down - - self.extensions = extensions - - def fqdn (self) : - if '.' in self.host : - return self.host + '.' - elif self.domain : - return pvl.dns.zone.fqdn(self.host, self.domain) - else : - raise ValueError("%s: have no fqdn/domain" % (self, )) - - def __str__ (self) : - return str(self.host) - -def apply_host_includes (options, config_path, include) : - """ - Yield files from a given config's include=... value - """ - - if options.hosts_include : - include_paths = (os.path.dirname(config_path), options.hosts_include) - else : - include_paths = (os.path.dirname(config_path), ) - - for include in include.split() : - for include_path in include_paths : - path = os.path.join(include_path, include) - if os.path.exists(path) : - break - else : - raise HostConfigError(config_path, "Unable to find include %s in include path: %s" % (include, ' '.join(include_paths))) - - if include.endswith('/') : - for name in os.listdir(path) : - file_path = os.path.join(path, name) - - if name.startswith('.') : - pass - elif not os.path.exists(file_path) : - log.warn("%s: skip nonexistant include: %s", config_path, file_path) - else : - log.info("%s: include: %s", config_path, file_path) - yield open(file_path) - - else : - log.info("%s: include: %s", config_path, path) - yield open(path) - -def apply_hosts_config (options, path, name, config, defaults={}, parent=None) : - """ - Load hosts from a ConfigObj section. - """ - - scalars = dict((scalar, config[scalar]) for scalar in config.scalars) - - if config.sections : - # includes? - for file in apply_host_includes(options, path, scalars.pop('include', '')) : - # within our domain context - for host in apply_hosts_file(options, file, parent=name) : - yield host - - # recurse; this is a domain meta-section - params = dict(defaults) - params.update(**scalars) # override - - log.debug("%s: %s -> %s", path, parent, name) - - for section in config.sections : - for host in apply_hosts_config(options, path, section, config[section], params, name) : - yield host - - elif name : - params = dict(defaults, **scalars) - - log.debug("%s: %s: %s", path, parent, name) - - # this is a host section - for host in Host.config(options, name, parent, **params) : - yield host - - else : - raise ValueError("No sections in config") - - -def apply_hosts_file (options, file, parent=None) : - """ - Load Hosts from a file. - - """ - - # use file basename as default - path = file.name - name = os.path.basename(path) - - try : - config = configobj.ConfigObj(file, - encoding = options.hosts_charset, - ) - except configobj.ParseError as ex : - raise HostConfigError("%s:%d: %s" % (path, ex.line_number, ex)) - - - return apply_hosts_config(options, path, name, config, parent=parent) - -def apply_hosts (options, files) : - """ - Load Hosts from files. - """ - - for file in files : - for host in apply_hosts_file(options, file) : - yield host - -def sort_hosts (options, hosts) : - """ - Yields hosts with a sorting key. - """ - - for host in hosts : - if host.ip : - sort = host.ip - else : - # sorts first - sort = ipaddr.IPAddress(0) - - yield sort, host - -def apply (options, args) : - """ - Load Hosts from arguments. - """ - - # without unicode - files = pvl.args.apply_files(args, 'r') - - # load configs - hosts = apply_hosts(options, files) - - # sort - hosts = list(sort_hosts(options, hosts)) - hosts.sort() - hosts = [host for sort, host in hosts] - - return hosts diff -r 6a8ea0d363c1 -r 1d755df7bf97 pvl/hosts/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pvl/hosts/__init__.py Tue Feb 24 17:35:36 2015 +0200 @@ -0,0 +1,9 @@ +__version__ = '0.8.0-dev' + +from pvl.hosts.config import ( + optparser, + apply, +) +from pvl.hosts.host import ( + Host, +) diff -r 6a8ea0d363c1 -r 1d755df7bf97 pvl/hosts/config.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pvl/hosts/config.py Tue Feb 24 17:35:36 2015 +0200 @@ -0,0 +1,272 @@ +""" + Load Hosts from config files. +""" + +import configobj +import logging; log = logging.getLogger('pvl.hosts') +import optparse +import os.path +import pvl.args +import sys + +from pvl.hosts.host import Host + +def optparser (parser): + hosts = optparse.OptionGroup(parser, "Hosts config files") + hosts.add_option('--hosts-charset', metavar='CHARSET', default='utf-8', + help="Encoding used for host files") + + hosts.add_option('--hosts-domain', metavar='DOMAIN', + help="Default domain for hosts. Default uses config file basename") + + hosts.add_option('--hosts-include', metavar='PATH', + help="Optional path for hosts includes, in addition to the host config dir") + + return hosts + +class HostConfigError (Exception): + """ + Generic error for file path. + """ + + def __init__ (self, config, error): + self.config = config + self.error = error + + def __str__ (self): + return "{self.config}: {self.error}".format(self=self) + +class HostConfigObjError (Exception): + """ + An error from ConfigObj for a config file path. + """ + + def __init__ (self, config, error): + self.config = config + self.error = error + + self.line_contents = error.line + + def __str__ (self): + return "{self.config}:{self.error.line_number}: {self.error.message}".format(self=self) + +def apply_host_expand (options, range, name, domain, **params): + """ + Expand a templated host item. + """ + + name = pvl.dns.zone.parse_generate_field(name) + + # expand all fields + params = {param: pvl.dns.zone.parse_generate_field(value) for param, value in params.iteritems()} + + for i in range: + yield Host.build(host(i), domain, + **{param: value(i) for param, value in params.iteritems()} + ) + +def parse_expand(name): + """ + Parse a name containing an optional expansion part. + + name: hostname containing optional "{...}" part + + Returns (name, range or None). + """ + + if '{' in name: + pre, name = name.split('{', 1) + range, post = name.rsplit('}', 1) + + range = pvl.dns.zone.parse_generate_range(range) + name = pre + "$" + post + else: + range = None + + return name, range + +def apply_host_config (options, path, parent, name, + domain=None, + **params +): + """ + Yield Hosts from a given config section. + + options global options + path config file for errors + parent parent filename/section containing this host item, or None. + used for domain + name name of the section (or file) containing this host item. + used for hostname + **params host parameters + """ + + if domain: + log.debug("%s: using explicit domain: %s", name, domain) + elif '.' in name: + log.debug("%s: using as fqdn without domain", name) + domain = None + elif parent: + log.debug("%s: default domain to section: %s", name, parent) + domain = parent + elif options.hosts_domain: + log.debug("%s: default domain to --hosts-domain: %s", name, options.hosts_domain) + domain = options.hosts_domain + else: + raise HostsConfigError(path, "{name}: no domain given".format(name=name)) + + # expand + name, range = parse_expand(name) + + if range: + for host in apply_host_expand(options, range, name, domain, **params): + yield host + else: + yield Host.build(name, domain, **params) + +def apply_host_includes (options, config_path, include): + """ + Yield files from a given config's include=... value + """ + + include_paths = [os.path.dirname(config_path)] + + if options.hosts_include: + include_paths.append[options.hosts_include] + + for include in include.split(): + for include_path in include_paths: + path = os.path.join(include_path, include) + if os.path.exists(path): + break + else: + raise HostConfigError(config_path, "Unable to find include {include} in include path: {include_path}".format( + include=include, + include_path=' '.join(include_paths), + )) + + if include.endswith('/'): + for name in os.listdir(path): + file_path = os.path.join(path, name) + + if name.startswith('.'): + pass + elif not os.path.exists(file_path): + log.warn("%s: skip nonexistant include: %s", config_path, file_path) + else: + log.info("%s: include: %s", config_path, file_path) + yield open(file_path) + + else: + log.info("%s: include: %s", config_path, path) + yield open(path) + +def apply_hosts_config (options, path, name, config, defaults={}, parent=None): + """ + Load hosts from a configobj.Section (which can be the top-level ConfigObj). + + options global options + path filesystem path of file (for errors) + name name of section/file + config configobj.Section + defaults hierarchial section defaults + parent optional parent section for included files + """ + + # items in this section + section = dict(defaults) + for scalar in config.scalars: + section[scalar] = config[scalar] + + if config.sections: + # this is a top-level section that includes hosts + + # process includes? + includes = section.pop('include', '') + + for file in apply_host_includes(options, path, includes): + # within our domain context + for host in apply_hosts_file(options, file, parent=name): + yield host + + # recurse until we hit a scalar-only section representing a host + for section_name in config.sections: + log.debug("%s: %s: %s", path, name, section_name) + + for host in apply_hosts_config(options, path, section_name, config[section_name], defaults=section, parent=name): + yield host + + elif parent: + # this is a host section + log.debug("%s: %s@%s", path, name, parent) + + for host in apply_host_config(options, path, parent, name, **section): + yield host + + else: + raise HostConfigError(path, "No sections in config") + +def apply_hosts_file (options, file, parent=None): + """ + Load Hosts from a file path. + + path:str filesystem path + parent included from given name + + Raises + HostConfigError + HostConfigObjError + """ + + # use file basename as default domain + path = file.name + name = os.path.basename(path) + + try: + config = configobj.ConfigObj(file, + raise_errors = True, # raise ConfigPObjError immediately + interpolation = 'Template', + encoding = options.hosts_charset, + ) + except configobj.ConfigObjError as ex: + raise HostConfigObjError(path, ex) + + return apply_hosts_config(options, path, name, config, parent=parent) + +def apply_hosts (options, files): + """ + Load Hosts from files. + + files:[str] list of filesystem paths + """ + + for path in files: + try: + file = open(path) + except IOError as ex: + raise HostConfigError(path, ex.strerror) + + for host in apply_hosts_file(options, file): + yield host + +def apply (options, args): + """ + Load Hosts from arguments. + """ + + try: + # load hosts from configs + hosts = list(apply_hosts(options, args)) + except HostConfigObjError as error: + log.error("%s", error) + log.error("\t%s", error.line_contents) + sys.exit(2) + + except HostConfigError as error: + log.error("%s", error) + sys.exit(2) + + # stable ordering + hosts = sorted(hosts, key=Host.sort_key) + + return hosts diff -r 6a8ea0d363c1 -r 1d755df7bf97 pvl/hosts/host.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pvl/hosts/host.py Tue Feb 24 17:35:36 2015 +0200 @@ -0,0 +1,205 @@ +import collections +import ipaddr +import logging; log = logging.getLogger('pvl.hosts.host') +import pvl.dns.zone + +class HostError (Exception) : + def __init__(self, name, error): + self.name = name + self.error = error + + def __str__ (self): + return "{self.name}: {self.error}".format(self=self) + +def parse_ethernet (value) : + """ + Normalize ethernet str. + """ + + return ':'.join('%02x' % int(x, 16) for x in value.split(':')) + +class Host (object) : + """ + A host is a network node that can have multiple ethernet interfaces, and multiple IP addresses in different domains. + """ + + # the label used for alias4/6 hosts + ALIAS4_FMT = '{host}-ipv4' + ALIAS6_FMT = '{host}-ipv6' + + @classmethod + def build (cls, name, domain, + ip=None, ip6=None, owner=None, location=None, + alias=None, alias4=None, alias6=None, + forward=None, reverse=None, + down=None, + boot=None, + **extra) : + """ + Return a Host initialized from data attributes. + + This handles all string parsing to our data types. + """ + + if alias : + alias = alias.split() + else : + alias = () + + if alias4 : + alias4 = alias4.split() + else : + alias4 = () + + if alias6 : + alias6 = alias6.split() + else : + alias6 = () + + ethernet = { } + extensions = collections.defaultdict(dict) + + for field, value in extra.iteritems() : + if ':' in field : + extension, field = field.split(':', 1) + + extensions[extension][field] = value + + continue + + if '.' in field : + field, instance = field.split('.') + else : + instance = None + + if field == 'ethernet' : + if instance : + ethernet[instance] = parse_ethernet(value) + else : + for eth in value.split() : + ethernet[len(ethernet)] = parse_ethernet(eth) + else : + raise HostError(name, "Unknown host field: %s=%s" % (field, value)) + + + if forward is None : + # normal zone + pass + elif forward : + # alias to external zone + pass + else : + # omit + forward = False + + if reverse is None : + # normal zone + pass + elif reverse : + # alias to external zone + pass + else : + # omit + reverse = False + + if down : + down = True + else : + down = None + + if not location : + location = location_domain = None + elif '@' in location: + location, location_domain = location.split('@', 1) + else: + location_domain = None + + return cls(name, + domain = domain, + ip = ipaddr.IPv4Address(ip) if ip else None, + ip6 = ipaddr.IPv6Address(ip6) if ip6 else None, + ethernet = ethernet, + alias = alias, + alias4 = alias4, + alias6 = alias6, + owner = owner, + location = location, + location_domain = location_domain, + boot = boot, + forward = forward, + reverse = reverse, + down = down, + + **extensions + ) + + def __init__ (self, host, + domain=None, + ip=None, ip6=None, + ethernet={ }, + alias=(), + owner=None, + location=None, + location_domain=None, + boot=None, + alias4=None, alias6=None, + forward=None, reverse=None, + down=None, + **extensions + ) : + """ + host - str + domain - str + ip - ipaddr.IPv4Address + ip6 - ipaddr.IPv6Address + ethernet - { index: ethernet } + alias - list + owner - str: LDAP uid + location - location name@ part + location_domain - location @domain part + alias4 - list (CNAME -> A) + alias6 - list (CNAME -> AAAA) + forward - generate forward records, or CNAME into given zone + reverse - generate reverse records, or CNAME into given zone + down - not online + """ + + self.host = host + self.domain = domain + self.ip = ip + self.ip6 = ip6 + self.ethernet = ethernet + self.alias = alias + self.alias4 = alias4 + self.alias6 = alias6 + self.owner = owner + self.location = location + self.location_domain = location_domain + self.boot = boot + self.forward = forward + self.reverse = reverse + self.down = down + + self.extensions = extensions + + def sort_key (self): + """ + Stable sort ordering + """ + + if self.ip : + return host.ip + else : + # sorts first + return ipaddr.IPAddress(0) + + def fqdn (self) : + if '.' in self.host : + return self.host + '.' + elif self.domain : + return pvl.dns.zone.fqdn(self.host, self.domain) + else : + raise ValueError("%s: have no fqdn/domain" % (self, )) + + def __str__ (self) : + return str(self.host) diff -r 6a8ea0d363c1 -r 1d755df7bf97 pvl/hosts/tests.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pvl/hosts/tests.py Tue Feb 24 17:35:36 2015 +0200 @@ -0,0 +1,46 @@ +import ipaddr +import unittest + +from pvl.hosts import config +from StringIO import StringIO + +class Options(object): + hosts_charset = 'utf-8' + hosts_domain = None + hosts_include = None + +class NamedStringIO(StringIO): + def __init__(self, name, buffer): + StringIO.__init__(self, buffer) + self.name = name + +class TestConfig(unittest.TestCase): + def setUp(self): + self.options = Options() + + def testApplyHostsError(self): + with self.assertRaises(config.HostConfigError): + list(config.apply_hosts(self.options, ['nonexistant'])) + + def testApplyHosts(self): + expected = [ + ('foo', 'test', ipaddr.IPAddress('127.0.0.1')), + ('bar', 'test', ipaddr.IPAddress('127.0.0.2')), + ] + + for expect, host in zip(expected, config.apply_hosts_file(self.options, NamedStringIO('test', """ +[foo] + ip = 127.0.0.1 + +[bar] + ip = 127.0.0.2 +""" + ))): + hostname, domain, ip = expect + + self.assertEquals(str(host), hostname) + self.assertEquals(host.domain, domain) + self.assertEquals(host.ip, ip) + +if __name__ == '__main__': + unittest.main() diff -r 6a8ea0d363c1 -r 1d755df7bf97 setup.py --- a/setup.py Tue Feb 24 14:53:50 2015 +0200 +++ b/setup.py Tue Feb 24 17:35:36 2015 +0200 @@ -14,7 +14,7 @@ # TODO: fix to use PEP-396 once available: # https://www.python.org/dev/peps/pep-0396/#classic-distutils -for line in open('pvl/hosts.py'): +for line in open('pvl/hosts/__init__.py'): if '__version__' in line: _, line_version = line.split('=') __version__ = line_version.strip().strip("''")