pvl.hosts: refactor as a package; cleanup pvl.hosts.config with some basic tests
--- a/pvl/hosts.py Tue Feb 24 14:53:50 2015 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,404 +0,0 @@
-"""
- Host definitions.
-"""
-
-import pvl.args
-import pvl.dns.zone
-
-import collections
-import configobj
-import ipaddr
-import optparse
-import os.path
-
-import logging; log = logging.getLogger('pvl.hosts')
-
-__version__ = '0.8.0-dev'
-
-def optparser (parser) :
- hosts = optparse.OptionGroup(parser, "Hosts input")
- hosts.add_option('--hosts-charset', metavar='CHARSET', default='utf-8',
- help="Encoding used for host files")
-
- hosts.add_option('--hosts-domain', metavar='DOMAIN',
- help="Default domain for hosts")
-
- hosts.add_option('--hosts-include', metavar='PATH',
- help="Optional path for hosts includes, beyond host config dir")
-
- return hosts
-
-class HostError (Exception) :
- pass
-
-class HostConfigError (HostError) :
- pass
-
-def parse_ethernet (value) :
- """
- Normalize ethernet str.
- """
-
- return ':'.join('%02x' % int(x, 16) for x in value.split(':'))
-
-class Host (object) :
- # the label used for alias4/6 hosts
- ALIAS4_FMT = '{host}-ipv4'
- ALIAS6_FMT = '{host}-ipv6'
-
- @classmethod
- def expand (cls, options, range, host, domain, ip, forward=None, reverse=None, **opts) :
- host = pvl.dns.zone.parse_generate_field(host)
- ip = pvl.dns.zone.parse_generate_field(ip)
-
- if forward is not None :
- _forward = pvl.dns.zone.parse_generate_field(forward)
- else :
- _forward = lambda i: forward
-
- if reverse is not None :
- _reverse = pvl.dns.zone.parse_generate_field(reverse)
- else :
- _reverse = lambda i: reverse
-
- for i in range :
- yield cls.build(options, host(i), domain,
- ip = ip(i),
- forward = _forward(i),
- reverse = _reverse(i),
- **opts
- )
-
- @classmethod
- def config (cls, options, host, section=None, domain=None, ip=None, **extra) :
- """
- Yield Hosts from a config section's scalars.
-
- options - pvl.args Options
- host - the name of the section (or file) containing this host item.
- section - the parent section containing this host item, or None
- used for domain
- """
-
- if domain :
- log.debug("%s: explicit domain: %s", host, domain)
- elif '.' in host :
- log.debug("%s: using as fqdn without domain", host)
- domain = None
- elif options.hosts_domain :
- log.debug("%s: default domain to --hots-domain: %s", host, options.hosts_domain)
- domain = options.hosts_domain
- elif section :
- log.debug("%s: default domain to section: %s", host, section)
- domain = section
- else :
- raise ValueError("%s: no domain given" % (host, ))
-
- if '{' in host :
- pre, host = host.split('{', 1)
- range, post = host.rsplit('}', 1)
-
- range = pvl.dns.zone.parse_generate_range(range)
- host = pre + "$" + post
-
- for host in cls.expand(options, range, host, domain, ip, **extra) :
- yield host
- else :
- yield cls.build(options, host, domain, ip=ip, **extra)
-
- @classmethod
- def build (cls, options, host, domain,
- ip=None, ip6=None, owner=None, location=None,
- alias=None, alias4=None, alias6=None,
- forward=None, reverse=None,
- down=None,
- boot=None,
- **extra) :
- """
- Return a Host from a config section's scalars.
-
- """
-
- if alias :
- alias = alias.split()
- else :
- alias = ()
-
- if alias4 :
- alias4 = alias4.split()
- else :
- alias4 = ()
-
- if alias6 :
- alias6 = alias6.split()
- else :
- alias6 = ()
-
- ethernet = { }
- extensions = collections.defaultdict(dict)
-
- for field, value in extra.iteritems() :
- if ':' in field :
- extension, field = field.split(':', 1)
-
- extensions[extension][field] = value
-
- continue
-
- if '.' in field :
- field, instance = field.split('.')
- else :
- instance = None
-
- if field == 'ethernet' :
- if instance :
- ethernet[instance] = parse_ethernet(value)
- else :
- for eth in value.split() :
- ethernet[len(ethernet)] = parse_ethernet(eth)
- else :
- raise ValueError("%s: Unknown host field: %s=%s" % (host, field, value))
-
-
- if forward is None :
- # normal zone
- pass
- elif forward :
- # alias to external zone
- pass
- else :
- # omit
- forward = False
-
- if reverse is None :
- # normal zone
- pass
- elif reverse :
- # alias to external zone
- pass
- else :
- # omit
- reverse = False
-
- if down :
- down = True
- else :
- down = None
-
- if not location :
- location = location_domain = None
- elif '@' in location:
- location, location_domain = location.split('@', 1)
- else:
- location_domain = None
-
- return cls(host,
- domain = domain,
- ip = ipaddr.IPv4Address(ip) if ip else None,
- ip6 = ipaddr.IPv6Address(ip6) if ip6 else None,
- ethernet = ethernet,
- alias = alias,
- alias4 = alias4,
- alias6 = alias6,
- owner = owner,
- location = location,
- location_domain = location_domain,
- boot = boot,
- forward = forward,
- reverse = reverse,
- down = down,
-
- **extensions
- )
-
- def __init__ (self, host,
- domain=None,
- ip=None, ip6=None,
- ethernet={ },
- alias=(),
- owner=None,
- location=None,
- location_domain=None,
- boot=None,
- alias4=None, alias6=None,
- forward=None, reverse=None,
- down=None,
- **extensions
- ) :
- """
- host - str
- domain - str
- ip - ipaddr.IPv4Address
- ip6 - ipaddr.IPv6Address
- ethernet - { index: ethernet }
- alias - list
- owner - str: LDAP uid
- location - location name@ part
- location_domain - location @domain part
- alias4 - list (CNAME -> A)
- alias6 - list (CNAME -> AAAA)
- forward - generate forward records, or CNAME into given zone
- reverse - generate reverse records, or CNAME into given zone
- down - not online
- """
-
- self.host = host
- self.domain = domain
- self.ip = ip
- self.ip6 = ip6
- self.ethernet = ethernet
- self.alias = alias
- self.alias4 = alias4
- self.alias6 = alias6
- self.owner = owner
- self.location = location
- self.location_domain = location_domain
- self.boot = boot
- self.forward = forward
- self.reverse = reverse
- self.down = down
-
- self.extensions = extensions
-
- def fqdn (self) :
- if '.' in self.host :
- return self.host + '.'
- elif self.domain :
- return pvl.dns.zone.fqdn(self.host, self.domain)
- else :
- raise ValueError("%s: have no fqdn/domain" % (self, ))
-
- def __str__ (self) :
- return str(self.host)
-
-def apply_host_includes (options, config_path, include) :
- """
- Yield files from a given config's include=... value
- """
-
- if options.hosts_include :
- include_paths = (os.path.dirname(config_path), options.hosts_include)
- else :
- include_paths = (os.path.dirname(config_path), )
-
- for include in include.split() :
- for include_path in include_paths :
- path = os.path.join(include_path, include)
- if os.path.exists(path) :
- break
- else :
- raise HostConfigError(config_path, "Unable to find include %s in include path: %s" % (include, ' '.join(include_paths)))
-
- if include.endswith('/') :
- for name in os.listdir(path) :
- file_path = os.path.join(path, name)
-
- if name.startswith('.') :
- pass
- elif not os.path.exists(file_path) :
- log.warn("%s: skip nonexistant include: %s", config_path, file_path)
- else :
- log.info("%s: include: %s", config_path, file_path)
- yield open(file_path)
-
- else :
- log.info("%s: include: %s", config_path, path)
- yield open(path)
-
-def apply_hosts_config (options, path, name, config, defaults={}, parent=None) :
- """
- Load hosts from a ConfigObj section.
- """
-
- scalars = dict((scalar, config[scalar]) for scalar in config.scalars)
-
- if config.sections :
- # includes?
- for file in apply_host_includes(options, path, scalars.pop('include', '')) :
- # within our domain context
- for host in apply_hosts_file(options, file, parent=name) :
- yield host
-
- # recurse; this is a domain meta-section
- params = dict(defaults)
- params.update(**scalars) # override
-
- log.debug("%s: %s -> %s", path, parent, name)
-
- for section in config.sections :
- for host in apply_hosts_config(options, path, section, config[section], params, name) :
- yield host
-
- elif name :
- params = dict(defaults, **scalars)
-
- log.debug("%s: %s: %s", path, parent, name)
-
- # this is a host section
- for host in Host.config(options, name, parent, **params) :
- yield host
-
- else :
- raise ValueError("No sections in config")
-
-
-def apply_hosts_file (options, file, parent=None) :
- """
- Load Hosts from a file.
-
- """
-
- # use file basename as default
- path = file.name
- name = os.path.basename(path)
-
- try :
- config = configobj.ConfigObj(file,
- encoding = options.hosts_charset,
- )
- except configobj.ParseError as ex :
- raise HostConfigError("%s:%d: %s" % (path, ex.line_number, ex))
-
-
- return apply_hosts_config(options, path, name, config, parent=parent)
-
-def apply_hosts (options, files) :
- """
- Load Hosts from files.
- """
-
- for file in files :
- for host in apply_hosts_file(options, file) :
- yield host
-
-def sort_hosts (options, hosts) :
- """
- Yields hosts with a sorting key.
- """
-
- for host in hosts :
- if host.ip :
- sort = host.ip
- else :
- # sorts first
- sort = ipaddr.IPAddress(0)
-
- yield sort, host
-
-def apply (options, args) :
- """
- Load Hosts from arguments.
- """
-
- # without unicode
- files = pvl.args.apply_files(args, 'r')
-
- # load configs
- hosts = apply_hosts(options, files)
-
- # sort
- hosts = list(sort_hosts(options, hosts))
- hosts.sort()
- hosts = [host for sort, host in hosts]
-
- return hosts
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/pvl/hosts/__init__.py Tue Feb 24 17:35:36 2015 +0200
@@ -0,0 +1,9 @@
+__version__ = '0.8.0-dev'
+
+from pvl.hosts.config import (
+ optparser,
+ apply,
+)
+from pvl.hosts.host import (
+ Host,
+)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/pvl/hosts/config.py Tue Feb 24 17:35:36 2015 +0200
@@ -0,0 +1,272 @@
+"""
+ Load Hosts from config files.
+"""
+
+import configobj
+import logging; log = logging.getLogger('pvl.hosts')
+import optparse
+import os.path
+import pvl.args
+import sys
+
+from pvl.hosts.host import Host
+
+def optparser (parser):
+ hosts = optparse.OptionGroup(parser, "Hosts config files")
+ hosts.add_option('--hosts-charset', metavar='CHARSET', default='utf-8',
+ help="Encoding used for host files")
+
+ hosts.add_option('--hosts-domain', metavar='DOMAIN',
+ help="Default domain for hosts. Default uses config file basename")
+
+ hosts.add_option('--hosts-include', metavar='PATH',
+ help="Optional path for hosts includes, in addition to the host config dir")
+
+ return hosts
+
+class HostConfigError (Exception):
+ """
+ Generic error for file path.
+ """
+
+ def __init__ (self, config, error):
+ self.config = config
+ self.error = error
+
+ def __str__ (self):
+ return "{self.config}: {self.error}".format(self=self)
+
+class HostConfigObjError (Exception):
+ """
+ An error from ConfigObj for a config file path.
+ """
+
+ def __init__ (self, config, error):
+ self.config = config
+ self.error = error
+
+ self.line_contents = error.line
+
+ def __str__ (self):
+ return "{self.config}:{self.error.line_number}: {self.error.message}".format(self=self)
+
+def apply_host_expand (options, range, name, domain, **params):
+ """
+ Expand a templated host item.
+ """
+
+ name = pvl.dns.zone.parse_generate_field(name)
+
+ # expand all fields
+ params = {param: pvl.dns.zone.parse_generate_field(value) for param, value in params.iteritems()}
+
+ for i in range:
+ yield Host.build(host(i), domain,
+ **{param: value(i) for param, value in params.iteritems()}
+ )
+
+def parse_expand(name):
+ """
+ Parse a name containing an optional expansion part.
+
+ name: hostname containing optional "{...}" part
+
+ Returns (name, range or None).
+ """
+
+ if '{' in name:
+ pre, name = name.split('{', 1)
+ range, post = name.rsplit('}', 1)
+
+ range = pvl.dns.zone.parse_generate_range(range)
+ name = pre + "$" + post
+ else:
+ range = None
+
+ return name, range
+
+def apply_host_config (options, path, parent, name,
+ domain=None,
+ **params
+):
+ """
+ Yield Hosts from a given config section.
+
+ options global options
+ path config file for errors
+ parent parent filename/section containing this host item, or None.
+ used for domain
+ name name of the section (or file) containing this host item.
+ used for hostname
+ **params host parameters
+ """
+
+ if domain:
+ log.debug("%s: using explicit domain: %s", name, domain)
+ elif '.' in name:
+ log.debug("%s: using as fqdn without domain", name)
+ domain = None
+ elif parent:
+ log.debug("%s: default domain to section: %s", name, parent)
+ domain = parent
+ elif options.hosts_domain:
+ log.debug("%s: default domain to --hosts-domain: %s", name, options.hosts_domain)
+ domain = options.hosts_domain
+ else:
+ raise HostsConfigError(path, "{name}: no domain given".format(name=name))
+
+ # expand
+ name, range = parse_expand(name)
+
+ if range:
+ for host in apply_host_expand(options, range, name, domain, **params):
+ yield host
+ else:
+ yield Host.build(name, domain, **params)
+
+def apply_host_includes (options, config_path, include):
+ """
+ Yield files from a given config's include=... value
+ """
+
+ include_paths = [os.path.dirname(config_path)]
+
+ if options.hosts_include:
+ include_paths.append[options.hosts_include]
+
+ for include in include.split():
+ for include_path in include_paths:
+ path = os.path.join(include_path, include)
+ if os.path.exists(path):
+ break
+ else:
+ raise HostConfigError(config_path, "Unable to find include {include} in include path: {include_path}".format(
+ include=include,
+ include_path=' '.join(include_paths),
+ ))
+
+ if include.endswith('/'):
+ for name in os.listdir(path):
+ file_path = os.path.join(path, name)
+
+ if name.startswith('.'):
+ pass
+ elif not os.path.exists(file_path):
+ log.warn("%s: skip nonexistant include: %s", config_path, file_path)
+ else:
+ log.info("%s: include: %s", config_path, file_path)
+ yield open(file_path)
+
+ else:
+ log.info("%s: include: %s", config_path, path)
+ yield open(path)
+
+def apply_hosts_config (options, path, name, config, defaults={}, parent=None):
+ """
+ Load hosts from a configobj.Section (which can be the top-level ConfigObj).
+
+ options global options
+ path filesystem path of file (for errors)
+ name name of section/file
+ config configobj.Section
+ defaults hierarchial section defaults
+ parent optional parent section for included files
+ """
+
+ # items in this section
+ section = dict(defaults)
+ for scalar in config.scalars:
+ section[scalar] = config[scalar]
+
+ if config.sections:
+ # this is a top-level section that includes hosts
+
+ # process includes?
+ includes = section.pop('include', '')
+
+ for file in apply_host_includes(options, path, includes):
+ # within our domain context
+ for host in apply_hosts_file(options, file, parent=name):
+ yield host
+
+ # recurse until we hit a scalar-only section representing a host
+ for section_name in config.sections:
+ log.debug("%s: %s: %s", path, name, section_name)
+
+ for host in apply_hosts_config(options, path, section_name, config[section_name], defaults=section, parent=name):
+ yield host
+
+ elif parent:
+ # this is a host section
+ log.debug("%s: %s@%s", path, name, parent)
+
+ for host in apply_host_config(options, path, parent, name, **section):
+ yield host
+
+ else:
+ raise HostConfigError(path, "No sections in config")
+
+def apply_hosts_file (options, file, parent=None):
+ """
+ Load Hosts from a file path.
+
+ path:str filesystem path
+ parent included from given name
+
+ Raises
+ HostConfigError
+ HostConfigObjError
+ """
+
+ # use file basename as default domain
+ path = file.name
+ name = os.path.basename(path)
+
+ try:
+ config = configobj.ConfigObj(file,
+ raise_errors = True, # raise ConfigPObjError immediately
+ interpolation = 'Template',
+ encoding = options.hosts_charset,
+ )
+ except configobj.ConfigObjError as ex:
+ raise HostConfigObjError(path, ex)
+
+ return apply_hosts_config(options, path, name, config, parent=parent)
+
+def apply_hosts (options, files):
+ """
+ Load Hosts from files.
+
+ files:[str] list of filesystem paths
+ """
+
+ for path in files:
+ try:
+ file = open(path)
+ except IOError as ex:
+ raise HostConfigError(path, ex.strerror)
+
+ for host in apply_hosts_file(options, file):
+ yield host
+
+def apply (options, args):
+ """
+ Load Hosts from arguments.
+ """
+
+ try:
+ # load hosts from configs
+ hosts = list(apply_hosts(options, args))
+ except HostConfigObjError as error:
+ log.error("%s", error)
+ log.error("\t%s", error.line_contents)
+ sys.exit(2)
+
+ except HostConfigError as error:
+ log.error("%s", error)
+ sys.exit(2)
+
+ # stable ordering
+ hosts = sorted(hosts, key=Host.sort_key)
+
+ return hosts
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/pvl/hosts/host.py Tue Feb 24 17:35:36 2015 +0200
@@ -0,0 +1,205 @@
+import collections
+import ipaddr
+import logging; log = logging.getLogger('pvl.hosts.host')
+import pvl.dns.zone
+
+class HostError (Exception) :
+ def __init__(self, name, error):
+ self.name = name
+ self.error = error
+
+ def __str__ (self):
+ return "{self.name}: {self.error}".format(self=self)
+
+def parse_ethernet (value) :
+ """
+ Normalize ethernet str.
+ """
+
+ return ':'.join('%02x' % int(x, 16) for x in value.split(':'))
+
+class Host (object) :
+ """
+ A host is a network node that can have multiple ethernet interfaces, and multiple IP addresses in different domains.
+ """
+
+ # the label used for alias4/6 hosts
+ ALIAS4_FMT = '{host}-ipv4'
+ ALIAS6_FMT = '{host}-ipv6'
+
+ @classmethod
+ def build (cls, name, domain,
+ ip=None, ip6=None, owner=None, location=None,
+ alias=None, alias4=None, alias6=None,
+ forward=None, reverse=None,
+ down=None,
+ boot=None,
+ **extra) :
+ """
+ Return a Host initialized from data attributes.
+
+ This handles all string parsing to our data types.
+ """
+
+ if alias :
+ alias = alias.split()
+ else :
+ alias = ()
+
+ if alias4 :
+ alias4 = alias4.split()
+ else :
+ alias4 = ()
+
+ if alias6 :
+ alias6 = alias6.split()
+ else :
+ alias6 = ()
+
+ ethernet = { }
+ extensions = collections.defaultdict(dict)
+
+ for field, value in extra.iteritems() :
+ if ':' in field :
+ extension, field = field.split(':', 1)
+
+ extensions[extension][field] = value
+
+ continue
+
+ if '.' in field :
+ field, instance = field.split('.')
+ else :
+ instance = None
+
+ if field == 'ethernet' :
+ if instance :
+ ethernet[instance] = parse_ethernet(value)
+ else :
+ for eth in value.split() :
+ ethernet[len(ethernet)] = parse_ethernet(eth)
+ else :
+ raise HostError(name, "Unknown host field: %s=%s" % (field, value))
+
+
+ if forward is None :
+ # normal zone
+ pass
+ elif forward :
+ # alias to external zone
+ pass
+ else :
+ # omit
+ forward = False
+
+ if reverse is None :
+ # normal zone
+ pass
+ elif reverse :
+ # alias to external zone
+ pass
+ else :
+ # omit
+ reverse = False
+
+ if down :
+ down = True
+ else :
+ down = None
+
+ if not location :
+ location = location_domain = None
+ elif '@' in location:
+ location, location_domain = location.split('@', 1)
+ else:
+ location_domain = None
+
+ return cls(name,
+ domain = domain,
+ ip = ipaddr.IPv4Address(ip) if ip else None,
+ ip6 = ipaddr.IPv6Address(ip6) if ip6 else None,
+ ethernet = ethernet,
+ alias = alias,
+ alias4 = alias4,
+ alias6 = alias6,
+ owner = owner,
+ location = location,
+ location_domain = location_domain,
+ boot = boot,
+ forward = forward,
+ reverse = reverse,
+ down = down,
+
+ **extensions
+ )
+
+ def __init__ (self, host,
+ domain=None,
+ ip=None, ip6=None,
+ ethernet={ },
+ alias=(),
+ owner=None,
+ location=None,
+ location_domain=None,
+ boot=None,
+ alias4=None, alias6=None,
+ forward=None, reverse=None,
+ down=None,
+ **extensions
+ ) :
+ """
+ host - str
+ domain - str
+ ip - ipaddr.IPv4Address
+ ip6 - ipaddr.IPv6Address
+ ethernet - { index: ethernet }
+ alias - list
+ owner - str: LDAP uid
+ location - location name@ part
+ location_domain - location @domain part
+ alias4 - list (CNAME -> A)
+ alias6 - list (CNAME -> AAAA)
+ forward - generate forward records, or CNAME into given zone
+ reverse - generate reverse records, or CNAME into given zone
+ down - not online
+ """
+
+ self.host = host
+ self.domain = domain
+ self.ip = ip
+ self.ip6 = ip6
+ self.ethernet = ethernet
+ self.alias = alias
+ self.alias4 = alias4
+ self.alias6 = alias6
+ self.owner = owner
+ self.location = location
+ self.location_domain = location_domain
+ self.boot = boot
+ self.forward = forward
+ self.reverse = reverse
+ self.down = down
+
+ self.extensions = extensions
+
+ def sort_key (self):
+ """
+ Stable sort ordering
+ """
+
+ if self.ip :
+ return host.ip
+ else :
+ # sorts first
+ return ipaddr.IPAddress(0)
+
+ def fqdn (self) :
+ if '.' in self.host :
+ return self.host + '.'
+ elif self.domain :
+ return pvl.dns.zone.fqdn(self.host, self.domain)
+ else :
+ raise ValueError("%s: have no fqdn/domain" % (self, ))
+
+ def __str__ (self) :
+ return str(self.host)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/pvl/hosts/tests.py Tue Feb 24 17:35:36 2015 +0200
@@ -0,0 +1,46 @@
+import ipaddr
+import unittest
+
+from pvl.hosts import config
+from StringIO import StringIO
+
+class Options(object):
+ hosts_charset = 'utf-8'
+ hosts_domain = None
+ hosts_include = None
+
+class NamedStringIO(StringIO):
+ def __init__(self, name, buffer):
+ StringIO.__init__(self, buffer)
+ self.name = name
+
+class TestConfig(unittest.TestCase):
+ def setUp(self):
+ self.options = Options()
+
+ def testApplyHostsError(self):
+ with self.assertRaises(config.HostConfigError):
+ list(config.apply_hosts(self.options, ['nonexistant']))
+
+ def testApplyHosts(self):
+ expected = [
+ ('foo', 'test', ipaddr.IPAddress('127.0.0.1')),
+ ('bar', 'test', ipaddr.IPAddress('127.0.0.2')),
+ ]
+
+ for expect, host in zip(expected, config.apply_hosts_file(self.options, NamedStringIO('test', """
+[foo]
+ ip = 127.0.0.1
+
+[bar]
+ ip = 127.0.0.2
+"""
+ ))):
+ hostname, domain, ip = expect
+
+ self.assertEquals(str(host), hostname)
+ self.assertEquals(host.domain, domain)
+ self.assertEquals(host.ip, ip)
+
+if __name__ == '__main__':
+ unittest.main()
--- a/setup.py Tue Feb 24 14:53:50 2015 +0200
+++ b/setup.py Tue Feb 24 17:35:36 2015 +0200
@@ -14,7 +14,7 @@
# TODO: fix to use PEP-396 once available:
# https://www.python.org/dev/peps/pep-0396/#classic-distutils
-for line in open('pvl/hosts.py'):
+for line in open('pvl/hosts/__init__.py'):
if '__version__' in line:
_, line_version = line.split('=')
__version__ = line_version.strip().strip("''")