# HG changeset patch # User Tero Marttila # Date 1387276816 -7200 # Node ID 9332f21f5aa189bf3c92bc8c2ae968a4e3b8af4f # Parent b8ba5df799be96b541c1653dd887d99312b1e5ea pvl.hosts-import: support host domains, rework multi-value fields diff -r b8ba5df799be -r 9332f21f5aa1 bin/pvl.hosts-import --- a/bin/pvl.hosts-import Tue Dec 17 10:53:42 2013 +0200 +++ b/bin/pvl.hosts-import Tue Dec 17 12:40:16 2013 +0200 @@ -68,9 +68,6 @@ help="Dump out info on imported host comments") # defaults - parser.add_option('--hosts-domain', metavar='DOMAIN', - help="Default domain for hosts") - parser.add_option('--zone-unused', metavar='HOST', help="DNS name for unallocated hosts") @@ -81,6 +78,9 @@ parser.add_option('--output-prefix', metavar='PREFIX', help="Select hosts by ip prefix") + parser.add_option('--output-domain', metavar='DOMAIN', + help="Select hosts by domain") + # defaults parser.set_defaults( import_zone_hosts = [], @@ -142,36 +142,34 @@ type = { 'A': 'ip', 'AAAA': 'ip6' }[rr.type] - yield host, 'domain', domain - yield host, type, ipaddr.IPAddress(ip) + yield (host, domain), type, ipaddr.IPAddress(ip) if rr.comment : - yield host, 'comment', rr.comment - + yield (host, domain), 'comment', rr.comment elif rr.type == 'CNAME' : alias, = rr.data alias_host, alias_domain = import_zone_host_name(options, alias, rr.origin) if domain == alias_domain : - yield alias_host, 'alias', host + yield (alias_host, alias_domain), 'alias', host else : - yield alias_host, 'alias', pvl.dns.zone.join(host, domain) + yield (alias_host, alias_domain), 'alias', pvl.dns.zone.join(host, domain) elif rr.type == 'TXT' : txt, = rr.data - yield host, 'comment', txt + yield (host, domain), 'comment', txt else : log.warn("%s: unknown rr: %s", host, rr) -def import_dhcp_host (options, host, items) : +def import_dhcp_host (options, dhcp_host, items) : """ Yield host infos from a dhcp host ... { ... } """ - hostname = None + host_name = None ethernet = [] fixed_address = None @@ -190,42 +188,55 @@ option = args.pop(0) if option == 'host-name' : - hostname, = args + host_name, = args else : - log.warn("host %s: ignore unknown option: %s", host, option) + log.warn("host %s: ignore unknown option: %s", dhcp_host, option) elif item == 'next-sever' : boot_server, = args elif item == 'filename' : boot_filename, = args else : - log.warn("host %s: ignore unknown item: %s", host, item) - - # determine hostname - suffix = None - - if '-' in host : - hostname, suffix = host.rsplit('-', 1) - else : - hostname = host + log.warn("host %s: ignore unknown item: %s", dhcp_host, item) - if fixed_address and not re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) : - hostname, domain = fixed_address.split('.', 1) + # determine host + host = None + domain = None + suffix = None + + if not fixed_address : + log.warn("%s: fixed-address is missing, unable to determine hostname/domainname", dhcp_host) + elif re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) : + log.warn("%s: fixed-address is an IP, unable to determine hostname/domainname", dhcp_host) + else : + host, domain = fixed_address.split('.', 1) - if not (hostname or ethernet) : - log.warn("%s: no hostname/ethernet: %s/%s", host, hostname, ethernet) - return - - yield hostname, 'ethernet', ethernet - #if suffix : - # yield hostname, ('ethernet', suffix), ethernet + # XXX: not actually true... eh + if host and dhcp_host.lower() == host.lower() : + # do not split suffix from host + pass + elif host and '-' in dhcp_host : + dhcp_host, suffix = dhcp_host.rsplit('-', 1) + elif '-' in dhcp_host : + host, suffix = dhcp_host.rsplit('-', 1) + else : + host = dhcp_host + + if not (host or ethernet) : + log.warn("%s: no hostname/ethernet: %s/%s", dhcp_host, hostname, ethernet) + elif suffix : + log.info("%s: %s@%s: %s: %s", dhcp_host, host, domain, suffix, ethernet) + yield (host, domain), 'ethernet.{suffix}'.format(suffix=suffix), ethernet + else : + log.info("%s: %s@%s: %s", dhcp_host, host, domain, ethernet) + yield (host, domain), 'ethernet', ethernet if boot_server and boot_filename : - yield hostname, 'boot', "{server}:{filename}".format( + yield (host, domain), 'boot', "{server}:{filename}".format( server = boot_server, filename = boot_filename, ) elif boot_filename : - yield hostname, 'boot', "{filename}".format(filename=boot_filename) + yield (host, domain), 'boot', "{filename}".format(filename=boot_filename) def import_dhcp_hosts (options, file_name, blocks) : """ @@ -395,18 +406,18 @@ log.info("%s: %s (%s)", host, owner, comment) - yield 'comment-owner', comment + yield 'comment.owner', comment yield 'owner', owner, elif 'group' in info or 'owner' in info : log.warn("%s: unknown owner: %s", host, info) - yield 'comment-owner', "{group} / {owner}".format( + yield 'comment.owner', "{group} / {owner}".format( group = info.get('group', ''), owner = info.get('owner', ''), ) if info.get('host') : - yield 'comment-host', info['host'] + yield 'comment.host', info['host'] def process_hosts_comments (options, import_hosts) : """ @@ -434,56 +445,128 @@ for field, value in process_host_comments(options, host, fields) : yield host, field, value -def apply_hosts_import (options) : +def import_hosts_files (options, zone_files, dhcp_files) : """ Import host infos from given files. """ - for zone_file in options.import_zone_hosts: + for zone_file in zone_files: file = pvl.args.apply_file(zone_file, 'r', options.input_charset) for info in import_zone_hosts(options, file) : yield info - for dhcp_file in options.import_dhcp_hosts: + for dhcp_file in dhcp_files : file = pvl.args.apply_file(dhcp_file, 'r', options.input_charset) for info in import_dhcp_conf(options, file) : yield info -def import_hosts (options) : +def process_import_hosts (options, import_hosts) : """ - Import hosts from dns/dhcp. + Build hosts from imported fields. + + Yields (domain, host), { (field, ...): value } """ - - import_hosts = apply_hosts_import(options) - import_hosts = process_hosts_comments(options, import_hosts) # gather hosts = collections.defaultdict(lambda: collections.defaultdict(list)) - for host, field, value in import_hosts : - hosts[host][field].append(value) + for (host, domain), field, value in import_hosts : + hosts[domain, host][tuple(field.split('.'))].append(value) - return hosts.iteritems() + # process + for (domain, host), fields in hosts.iteritems() : + SINGLE_FIELDS = ( + 'ip', + 'ip6', + 'comment.owner', + 'owner', + 'boot', + ) + MULTI_FIELDS = ( + 'comment.host', + 'ethernet', + 'alias', + ) + host_fields = {} + + for field_name in SINGLE_FIELDS : + field = tuple(field_name.split('.')) + values = fields.get(field) + + if not values : + continue + elif len(values) == 1 : + value, = values + else : + log.error("%s@%s: multiple %s: %s", host, domain, field, values) + value = values[0] + + log.debug("%s@%s: %s: %s", host, domain, field, value) + host_fields[field] = value + + for field_name in MULTI_FIELDS : + field_prefix = tuple(field_name.split('.')) + + # find labled fields by prefix, or unlabled multi-fields + for field, values in fields.iteritems() : + pre, field_index = field[:-1], field[-1] + + if not values : + pass + + elif pre == field_prefix : + log.debug("%s@%s: %s.%s: %s", host, domain, field_prefix, field_index, value) + host_fields[field] = values + + elif field == field_prefix : + log.debug("%s@%s: %s.*: %s", host, domain, field_prefix, value) + host_fields[field_prefix] = values + + yield (host, domain), host_fields + +def apply_import_hosts (options) : + """ + Import hosts. + """ + + import_hosts = import_hosts_files(options, options.import_zone_hosts, options.import_dhcp_hosts) + + # process + import_hosts = process_hosts_comments(options, import_hosts) + + # gather + return process_import_hosts(options, import_hosts) def check_hosts (options, hosts) : by_name = dict(hosts) for host, fields in hosts : - if set(fields) == set(['alias']) : - log.warn("%s: nonexistant alias target: %s", host, ' '.join(fields['alias'])) + if set(fields) == set([('alias', )]) : + log.warn("%s: nonexistant alias target: %s", host, ' '.join(fields[('alias', )])) def sort_export_hosts (options, hosts) : + """ + Generate a sortable version of hosts, yielding (sort, host, fields). + """ + if options.output_prefix : prefix = ipaddr.IPNetwork(options.output_prefix) else : prefix = None - for host, fields in hosts : - ip = fields.get('ip') + if options.output_domain : + select_domain = options.output_domain + else : + select_domain = None + + for (host, domain), fields in hosts : + ip = fields.get(('ip', )) + + log.debug("%s@%s: ip=%s", host, domain, ip) # sort by IP if ip : - sort = ip[0] + sort = ip else : # fake, to sort correctly sort = ipaddr.IPAddress(0) @@ -493,50 +576,73 @@ if not (ip and ip in prefix) : continue - yield sort, host, fields + if select_domain : + if not (domain and domain == select_domain) : + continue + + yield (domain, sort), (host, domain), fields def export_hosts (options, hosts) : """ Generate hosts config lines for given hosts. """ - FMT = u"\t{field:15} = {value}" - - yield u"[{domain}]".format(domain=options.hosts_domain) - # filter + sort hosts = [(host, fields) for sort, host, fields in sorted(sort_export_hosts(options, hosts))] - for host, fields in hosts : - for comment in fields.get('comment-host', ()): - yield u"# {comment}".format(comment=comment) - - yield u"[[{host}]]".format(host=host) - - for domain in fields.get('domain', ()) : - if domain != options.hosts_domain : - yield FMT.format(field='domain', value=domain) + if options.output_domain : + # global + output_domain = False - for field, fmt in ( - ('ip', FMT), - ('ip6', FMT), - ('ethernet', FMT), - ('owner', u"\t{field:15} = {value} # {fields[comment-owner][0]}"), - ('alias', FMT), - ('boot', FMT), - ) : - values = fields.get(field, ()) + yield u"{field:15} = {domain}".format(field='domain', domain=options.output_domain) + yield u"" + else : + output_domain = None - if len(values) > 1 : - for index, value in enumerate(values, 1) : - yield fmt.format( - field = "{field}.{index}".format(field=field, index=index), + for (host, domain), fields in hosts : + if output_domain is False : + pass + elif domain != output_domain : + yield u"[{domain}]".format(domain=domain) + output_domain = domain + + # optional host-comments + for comment in fields.get(('comment', 'host'), ()): + yield u"{indent}# {comment}".format( + indent = '\t' if output_domain else '', + comment = comment, + ) + + if output_domain : + yield u"\t[[{host}]]".format(host=host) + else : + yield u"[{host}]".format(host=host) + + #if not options.output_domain and domain : + # yield u"\t{field:15} = {domain}".format(field='domain', domain=domain) + + for field_name in ( + 'ip', + 'ip6', + 'ethernet', + 'owner', + 'alias', + 'boot', + ) : + for field, value in fields.iteritems() : + if field[0] == field_name : + # optional field-comment + comment = fields.get(('comment', field_name), None) + + if isinstance(value, list) : + value = ' '.join(value) + + yield u"{indent}{field:15} = {value} {comment}".format( + indent = '\t\t' if output_domain else '\t', + field = '.'.join(str(label) for label in field), value = value, - fields = fields - ) - elif len(values) > 0 : - value, = values - yield fmt.format(field=field, value=value, fields=fields) + comment = u"# {comment}".format(comment=comment) if comment else '', + ).rstrip() yield "" @@ -555,12 +661,8 @@ options.ldap = pvl.ldap.args.apply(options) - if args : - # direct from file - hosts = pvl.args.apply_files(args, 'r', options.input_charset) - else : - # import - hosts = list(import_hosts(options)) + # import + hosts = list(apply_import_hosts(options)) # verify check_hosts(options, hosts)