"""
Load Hosts from config files.
"""
import configobj
import logging; log = logging.getLogger('pvl.hosts')
import optparse
import os.path
import pvl.args
import pvl.dns
import sys
from pvl.hosts.host import Host
def optparser (parser):
hosts = optparse.OptionGroup(parser, "Hosts config files")
hosts.add_option('--hosts-charset', metavar='CHARSET', default='utf-8',
help="Encoding used for host files")
hosts.add_option('--hosts-domain', metavar='DOMAIN',
help="Default domain for hosts. Default uses config file basename")
hosts.add_option('--hosts-include', metavar='PATH',
help="Optional path for hosts includes, in addition to the host config dir")
return hosts
class HostConfigError (Exception):
"""
Generic error for file path.
"""
def __init__ (self, config, error):
self.config = config
self.error = error
def __str__ (self):
return "{self.config}: {self.error}".format(self=self)
class HostConfigObjError (Exception):
"""
An error from ConfigObj for a config file path.
"""
def __init__ (self, config, error):
self.config = config
self.error = error
self.line_contents = error.line
def __str__ (self):
return "{self.config}:{self.error.line_number}: {self.error.message}".format(self=self)
def parse_expand(name):
"""
Parse a name containing an optional expansion part.
name: hostname containing optional "{...}" part
Returns (name, range or None).
"""
if '{' in name:
# consume the first {...} token as the range
pre, name = name.split('{', 1)
range, post = name.split('}', 1)
# if there's a second {...} token, it will be re-composed into ${...}
name = pre + "$" + post
# TODO: raise HostConfigError
range = pvl.dns.parse_generate_range(range)
else:
range = None
return name, range
def parse_config_field(field):
"""
Parse structured config fields.
[<extension> ":"] <field> ["." <instance>]
"""
if ':' in field :
extension, field = field.split(':', 1)
else:
extension = None
if '.' in field :
field, instance = field.split('.')
else :
instance = None
return extension, field, instance
def apply_host (name, domain, config):
"""
Return Host from an (expanded) config section.
name - (expanded) name of host
domain - domain for host
config - host config fields to parse
Raises ValueError.
"""
fields = { }
extensions = fields['extensions'] = { }
for field, value in config.iteritems():
extension, field, instance = parse_config_field(field)
if extension:
f = extensions.setdefault(extension, {})
else:
f = fields
if not instance and field not in f:
f[field] = value
elif field not in f:
f[field] = {instance: value}
elif isinstance(f[field], dict):
f.setdefault(field, {})[instance] = value
else:
raise ValueError("override instanced {field} value: {value}".format(field=field, value=value))
return Host.build(name, domain, **fields)
def apply_hosts (parent, name, config):
"""
Yield Hosts from a given config section.
parent - parent filename/section containing this host item, or None.
used for domain
name - name of the section (or file) containing this host item.
used for hostname
config - host parameters to parse
Raises ValueError.
"""
if '@' in name:
name, domain = name.split('@', 1)
log.debug("%s: using explicit domain: %s", name, domain)
elif '.' in name:
log.debug("%s: using as fqdn without domain", name)
domain = None
elif parent:
log.debug("%s: default domain to section: %s", name, parent)
domain = parent
else:
# XXX: impossible?
raise ValueError("no domain given")
# expand?
name, range = parse_expand(name)
if range:
generate_name = pvl.dns.parse_generate_field(name)
# expand all fields
generate_config = {field: pvl.dns.parse_generate_field(value) for field, value in config.iteritems()}
for i in range:
yield apply_host(generate_name(i), domain,
{field: value(i) for field, value in generate_config.iteritems()},
)
else:
# single host
yield apply_host(name, domain, config)
def parse_config_includes (options, config_path, includes, **opts):
"""
Yield file paths from a given config's include=... value
"""
# relative
include_paths = [os.path.dirname(config_path)]
if options.hosts_include:
include_paths.append(options.hosts_include)
for include in includes:
for include_path in include_paths:
path = os.path.join(include_path, include)
if os.path.exists(path):
break
else:
raise HostConfigError(config_path, "Unable to find include {include} in include path: {include_path}".format(
include=include,
include_path=' '.join(include_paths),
))
log.info("%s: include: %s", config_path, path)
yield path
def apply_hosts_configs (options, path, name, config, parent=None, defaults={}):
"""
Load hosts from a configobj.Section (which can be the top-level ConfigObj).
options global options
path filesystem path of file (for errors)
name name of this section/file
config configobj.Section
parent parent section from included files or --hosts-domain
defaults hierarchial section defaults
"""
# items in this section
section = dict(defaults)
for scalar in config.scalars:
section[scalar] = config[scalar]
# process includes?
if 'include' in section:
includes = section.pop('include').split()
includes = list(parse_config_includes(options, path, includes))
# within our domain context
for host in apply_hosts_files(options, includes, parent=name):
yield host
else:
includes = None
if config.sections:
# this is a top-level section that includes hosts
if parent:
log.info("%s: @%s@%s", path, name, parent)
name = pvl.dns.join(name, parent)
else:
log.info("%s: @%s", path, name)
# recurse until we hit a scalar-only section representing a host
for section_name in config.sections:
log.debug("%s: %s: %s", path, name, section_name)
for host in apply_hosts_configs(options, path, section_name, config[section_name], parent=name, defaults=section):
yield host
elif parent:
# this is a host section
log.debug("%s: %s@%s", path, name, parent)
try:
for host in apply_hosts(parent, name, section):
log.info("%s: %s", path, host)
yield host
except ValueError as error:
log.exception("%s: %s: %s", path, parent, name)
raise HostConfigError(path, "{parent}: {name}: {error}".format(parent=parent, name=name, error=error))
elif includes:
# includes-only zone
pass
elif section:
raise HostConfigError(path, "Top-level hosts are only allowed in included confs")
else:
# empty file
log.info("%s: skip empty conf", path)
def apply_hosts_config (options, file, **opts):
"""
Load Hosts from a file path.
file - opened file object, with .name attribute
Raises
HostConfigError
HostConfigObjError
"""
# use file basename as default domain
path = file.name
name = os.path.basename(path)
try:
config = configobj.ConfigObj(file,
raise_errors = True, # raise ConfigPObjError immediately
interpolation = 'Template',
encoding = options.hosts_charset,
)
except configobj.ConfigObjError as ex:
raise HostConfigObjError(path, ex)
return apply_hosts_configs(options, path, name, config, **opts)
def apply_hosts_file (options, path, **opts):
"""
Load Hosts from a file path.
"""
try:
file = open(path)
except IOError as ex:
raise HostConfigError(path, ex.strerror)
for host in apply_hosts_config(options, file, **opts):
yield host
def apply_hosts_directory (options, root, **opts):
"""
Load Hosts from a directory, loading each file within the directory.
Skips .dotfiles.
"""
for name in os.listdir(root):
path = os.path.join(root, name)
if name.startswith('.'):
log.debug("%s: skip dotfile: %s", root, name)
continue
if os.path.isdir(path):
log.debug("%s: skip directory: %s", root, name)
continue
for host in apply_hosts_file(options, path, **opts):
yield host
def apply_hosts_files (options, files, **opts):
"""
Load Hosts from files.
files:[str] list of filesystem paths, which may be directories or files
"""
for path in files:
if os.path.isdir(path):
for host in apply_hosts_directory(options, path, **opts):
yield host
else:
for host in apply_hosts_file(options, path, **opts):
yield host
def apply (options, args):
"""
Load Hosts from arguments.
Exits with status=2 if loading the confs fails.
"""
try:
# load hosts from configs
hosts = list(apply_hosts_files(options, args))
except HostConfigObjError as error:
log.error("%s", error)
log.error("\t%s", error.line_contents)
sys.exit(2)
except HostConfigError as error:
log.error("%s", error)
sys.exit(2)
# stable ordering
return sorted(hosts, key=Host.sort_key)