terom@259: #!/usr/bin/env python
terom@259: 
terom@259: """
terom@268:     Import hosts from existing BIND or dhcpd files.
terom@259: """
terom@259: 
terom@268: import pvl.args
terom@259: import pvl.dns.zone
terom@259: import pvl.dhcp.config
terom@313: import pvl.hosts
terom@261: import pvl.ldap.args
terom@259: 
terom@302: import collections
terom@263: import ipaddr
terom@302: import logging; log = logging.getLogger('pvl.hosts-import')
terom@263: import optparse
terom@302: import os.path
terom@259: import re
terom@259: 
terom@259: __version__ = '0.1'
terom@259: 
terom@259: def parse_options (argv) :
terom@259:     """
terom@259:         Parse command-line arguments.
terom@259:     """
terom@259: 
terom@259:     parser = optparse.OptionParser(
terom@259:             prog        = argv[0],
terom@259:             usage       = '%prog: [options]',
terom@259:             version     = __version__,
terom@259: 
terom@259:             # module docstring
terom@259:             description = __doc__,
terom@259:     )
terom@259: 
terom@259:     # logging
terom@259:     parser.add_option_group(pvl.args.parser(parser))
terom@261:     parser.add_option_group(pvl.ldap.args.parser(parser))
terom@259: 
terom@259:     parser.add_option('-c', '--input-charset',  metavar='CHARSET',  default='utf-8', 
terom@259:             help="Encoding used for input files")
terom@259: 
terom@259:     parser.add_option('--output-charset',       metavar='CHARSET',  default='utf-8', 
terom@259:             help="Encoding used for output files")
terom@259: 
terom@259:     # input
terom@301:     parser.add_option('--import-zone-hosts',    metavar='FILE',     action='append',
terom@259:             help="Load hosts from DNS zone")
terom@259: 
terom@302:     parser.add_option('--import-zone-origin',   metavar='ORIGIN',
terom@302:             help="Initial origin for given zone file; default is basename")
terom@280: 
terom@301:     parser.add_option('--import-zone-comments-owner',  action='store_const',
terom@301:             dest='import_zone_comments', const='owner',
terom@275:             help="Import DNS zone comment as owner comment")
terom@275: 
terom@301:     parser.add_option('--import-zone-comments-host',   action='store_const',
terom@301:             dest='import_zone_comments', const='host',
terom@275:             help="Import DNS zone comment as host comment")
terom@275: 
terom@302:     parser.add_option('--import-dhcp-hosts',    metavar='FILE',     action='append',
terom@302:             help="Load hosts from DHCP config")
terom@302: 
terom@302:     parser.add_option('--import-dhcp-boot-server',      metavar='NEXT-SERVER',
terom@302:             help="Default boot_server for dpc hosts")
terom@302: 
terom@302: 
terom@261:     parser.add_option('--dump-host-comments',   action='store_true',
terom@261:             help="Dump out info on imported host comments")
terom@261: 
terom@259:     # defaults
terom@259:     parser.add_option('--zone-unused',          metavar='HOST',
terom@259:             help="DNS name for unallocated hosts")
terom@259: 
terom@259:     # output
terom@311:     parser.add_option('--output-hosts',         metavar='FILE',         default='-',
terom@259:             help="Output hosts file")
terom@259: 
terom@265:     parser.add_option('--output-prefix',        metavar='PREFIX',
terom@265:             help="Select hosts by ip prefix")
terom@263: 
terom@304:     parser.add_option('--output-domain',        metavar='DOMAIN',
terom@304:             help="Select hosts by domain")
terom@304: 
terom@327:     parser.add_option('--output-others',        action='store_true',
terom@327:             help="Negate selection")
terom@327: 
terom@259:     # defaults
terom@259:     parser.set_defaults(
terom@301:         import_zone_hosts   = [],
terom@301:         import_dhcp_hosts   = [],
terom@259:     )
terom@259:     
terom@259:     # parse
terom@259:     options, args = parser.parse_args(argv[1:])
terom@259: 
terom@259:     # apply
terom@259:     pvl.args.apply(options, argv[0])
terom@259: 
terom@259:     return options, args
terom@259: 
terom@302: def import_zone_host_name (options, name, origin) :
terom@302:     """
terom@302:         Import zone name from rr
terom@302:     """
terom@302: 
terom@302:     if '.' in name :
terom@302:         host, domain = name.split('.', 1)
terom@309:         domain = pvl.dns.join(domain, origin)
terom@302:     else :
terom@302:         host = name
terom@302:         domain = origin
terom@302:             
terom@302:     if domain :
terom@302:         # not a fqdn
terom@302:         domain = domain.rstrip('.')
terom@302: 
terom@302:         log.info("%s: %s@%s", name, host, domain)
terom@302:     else :
terom@302:         log.warn("%s: no domain", name)
terom@302: 
terom@302:     return host, domain
terom@302: 
terom@263: def import_zone_hosts (options, file) :
terom@259:     """
terom@259:         Yield host info from zonefile records.
terom@259:     """
terom@259: 
terom@302:     origin = options.import_zone_origin or os.path.basename(file.name)
terom@302: 
terom@318:     for line, rr in pvl.dns.zone.ZoneLine.load(file,
terom@302:             # used to determine domain
terom@302:             origin          = origin,
terom@302: 
terom@318:             # lazy-import generated hosts on demand
terom@318:             expand_generate = True,
terom@279:     ) :
terom@319:         if rr :
terom@319:             pass
terom@319:         elif line.parts[0] == '$ORIGIN' :
terom@319:             # handled by ZoneLine.load
terom@319:             continue
terom@319:         else :
terom@318:             log.warn("%s: skip non-rr line: %s", line, line.line)
terom@318:             continue
terom@318: 
terom@302:         host, domain = import_zone_host_name(options, rr.name, rr.origin)
terom@302: 
terom@328:         if rr.type in ('A', 'AAAA') :
terom@259:             ip, = rr.data
terom@328:             ip = ipaddr.IPAddress(ip)
terom@259: 
terom@281:             type = { 'A': 'ip', 'AAAA': 'ip6' }[rr.type]
terom@281: 
terom@328:             if options.zone_unused and rr.name == options.zone_unused :
terom@328:                 yield (str(ip), domain), 'ip.unused', ip
terom@328:             else :
terom@328:                 yield (host, domain), type, ip,
terom@259: 
terom@259:             if rr.comment :
terom@304:                 yield (host, domain), 'comment', rr.comment
terom@275: 
terom@318:             if line.parts[0] == '$GENERATE' :
terom@318:                 # only import as host if used for other purposes as well
terom@318:                 yield (host, domain), 'lazy-import', True
terom@318: 
terom@259:         elif rr.type == 'CNAME' :
terom@302:             alias, = rr.data
terom@302:             alias_host, alias_domain = import_zone_host_name(options, alias, rr.origin)
terom@302:             
terom@302:             if domain == alias_domain :
terom@304:                 yield (alias_host, alias_domain), 'alias', host
terom@325:             elif domain.endswith('.' + alias_domain) :
terom@325:                 yield (alias_host, alias_domain), 'alias', pvl.dns.join(host, domain[:(len(domain) - len(alias_domain) - 1)])
terom@302:             else :
terom@325:                 log.warn("%s@%s: alias outside of target domain: %s", host, domain, alias_domain)
terom@301:         
terom@301:         elif rr.type == 'TXT' :
terom@301:             txt, = rr.data
terom@301: 
terom@304:             yield (host, domain), 'comment', txt
terom@259: 
terom@259:         else :
terom@302:             log.warn("%s: unknown rr: %s", host, rr)
terom@259: 
terom@304: def import_dhcp_host (options, dhcp_host, items) :
terom@259:     """
terom@259:         Yield host infos from a dhcp host ... { ... }
terom@259:     """
terom@259: 
terom@304:     host_name = None
terom@259:     ethernet = []
terom@259:     fixed_address = None
terom@259: 
terom@280:     boot_server = options.import_dhcp_boot_server
terom@280:     boot_filename = None
terom@280: 
terom@259:     for item in items :
terom@259:         item, args = item[0], item[1:]
terom@259: 
terom@259:         if item == 'hardware' :
terom@259:             _ethernet, ethernet = args
terom@259:             assert _ethernet == 'ethernet'
terom@259:         elif item == 'fixed-address' :
terom@259:             fixed_address, = args
terom@259:         elif item == 'option' :
terom@259:             option = args.pop(0)
terom@259: 
terom@259:             if option == 'host-name' :
terom@304:                 host_name, = args
terom@259:             else :
terom@304:                 log.warn("host %s: ignore unknown option: %s", dhcp_host, option)
terom@280:         elif item == 'next-sever' :
terom@280:             boot_server, = args
terom@280:         elif item == 'filename' :
terom@280:             boot_filename, = args
terom@259:         else :
terom@304:             log.warn("host %s: ignore unknown item: %s", dhcp_host, item)
terom@259: 
terom@304:     # determine host
terom@304:     host = None
terom@304:     domain = None
terom@304:     suffix = None
terom@304:     
terom@304:     if not fixed_address :
terom@304:         log.warn("%s: fixed-address is missing, unable to determine hostname/domainname", dhcp_host)
terom@304:     elif re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :
terom@304:         log.warn("%s: fixed-address is an IP, unable to determine hostname/domainname", dhcp_host)
terom@304:     else :
terom@304:         host, domain = fixed_address.split('.', 1)
terom@264: 
terom@304:     # XXX: not actually true... eh
terom@304:     if host and dhcp_host.lower() == host.lower() :
terom@304:         # do not split suffix from host
terom@304:         pass
terom@304:     elif host and '-' in dhcp_host :
terom@304:         dhcp_host, suffix = dhcp_host.rsplit('-', 1)
terom@304:     elif '-' in dhcp_host :
terom@304:         host, suffix = dhcp_host.rsplit('-', 1)
terom@304:     else :
terom@304:         host = dhcp_host
terom@304: 
terom@304:     if not (host or ethernet) :
terom@304:         log.warn("%s: no hostname/ethernet: %s/%s", dhcp_host, hostname, ethernet)
terom@304:     elif suffix :
terom@304:         log.info("%s: %s@%s: %s: %s", dhcp_host, host, domain, suffix, ethernet)
terom@304:         yield (host, domain), 'ethernet.{suffix}'.format(suffix=suffix), ethernet
terom@304:     else :
terom@304:         log.info("%s: %s@%s: %s", dhcp_host, host, domain, ethernet)
terom@304:         yield (host, domain), 'ethernet', ethernet
terom@280: 
terom@280:     if boot_server and boot_filename :
terom@304:         yield (host, domain), 'boot', "{server}:{filename}".format(
terom@280:                 server      = boot_server,
terom@280:                 filename    = boot_filename,
terom@280:         )
terom@280:     elif boot_filename :
terom@304:         yield (host, domain), 'boot', "{filename}".format(filename=boot_filename)
terom@259: 
terom@303: def import_dhcp_hosts (options, file_name, blocks) :
terom@259:     """
terom@259:         Process hosts from a parsed block
terom@259:     """
terom@259: 
terom@259:     for block, items, blocks in blocks :
terom@259:         
terom@259:         block, args = block[0], block[1:]
terom@259: 
terom@259:         if block == 'group' :
terom@303:             log.info("%s: group", file_name)
terom@303:             for info in import_dhcp_hosts(options, file_name, blocks) :
terom@259:                 yield info
terom@259:         elif block == 'host' :
terom@259:             host, = args
terom@301:             
terom@303:             log.info("%s: host: %s", file_name, host)
terom@259: 
terom@259:             try :
terom@263:                 for info in import_dhcp_host(options, host, items) :
terom@259:                     yield info
terom@259:             except ValueError as error :
terom@303:                 log.exception("%s: invalid host %s: %s", file_name, host, error)
terom@259:         else:
terom@303:             log.warn("%s: ignore unknown block: %s", file_name, block)
terom@259: 
terom@263: def import_dhcp_conf (options, file) :
terom@259:     items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)
terom@259: 
terom@259:     for item in items :
terom@259:         item, args = item[0], item[1:]
terom@259: 
terom@259:         if item == 'include' :
terom@259:             include, = args
terom@263:             for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :
terom@259:                 yield info
terom@259:         else :
terom@259:             log.warn("ignore unknown item: %s", item)
terom@259:     
terom@303:     for info in import_dhcp_hosts(options, file.name, blocks) :
terom@259:         yield info
terom@259: 
terom@260: ZONE_COMMENTS = (
terom@260:         re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),
terom@260:         re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),
terom@260:         re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),
terom@260:         re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),
terom@260: )
terom@260: 
terom@260: ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')
terom@260: 
terom@260: def process_zone_comment (options, hostname, comment) :
terom@260:     """
terom@260:         Attempt to parse a host comment field... :D
terom@260: 
terom@260:         Yields (field, value) bits
terom@260:     """
terom@260: 
terom@260:     for regex in ZONE_COMMENTS :
terom@260:         match = regex.match(comment)
terom@260: 
terom@260:         if match :
terom@275:             matches = match.groupdict()
terom@279: 
terom@279:             log.info("%s: matched comment: %s", hostname, comment)
terom@260:             break
terom@260:     else :
terom@301:         if options.import_zone_comments :
terom@279:             log.info("%s: default comment: %s", hostname, comment)
terom@301:             matches = { options.import_zone_comments: comment }
terom@275:         else :
terom@275:             log.warn("%s: unknown comment: %s", hostname, comment)
terom@275:             return
terom@260:     
terom@260:     owner = matches.pop('owner', None)
terom@260:     
terom@260:     if owner :
terom@260:         mail_match = ZONE_OWNER_MAIL.match(owner)
terom@260: 
terom@260:         if mail_match :
terom@260:             mail_matches = mail_match.groupdict()
terom@260:             
terom@260:             owner = mail_matches['owner']
terom@261:             yield 'mail', mail_matches['mail'].strip()
terom@275:     
terom@275:         yield 'owner', owner.strip()
terom@260: 
terom@260:     for field, value in matches.iteritems() :
terom@260:         if value :
terom@261:             yield field, value.strip()
terom@261: 
terom@263: NONE_OWNERS = set((
terom@263:     u'tech',
terom@263:     u'atk',
terom@263:     u'toimisto',
terom@263: ))
terom@261: 
terom@263: def process_host_owner_ldap (options, host, info) :
terom@261:     """
terom@261:         Yield guesses for user from LDAP.
terom@261:     """
terom@261: 
terom@261:     if info.get('mail') :
terom@261:         for user in options.ldap.users.filter(
terom@261:                 { 'mailLocalAddress': info['mail'] },
terom@261:                 { 'uid': info['mail'] },
terom@261:         ) :
terom@263:             yield user, None
terom@261: 
terom@261:     if info.get('group') and info.get('owner') :
terom@261:         groups = options.ldap.groups.filter(cn=info['group'])
terom@261: 
terom@261:         for group in groups :
terom@261:             for user in options.ldap.users.filter({
terom@261:                 'gidNumber': group['gidNumber'],
terom@261:                 'cn': info['owner'],
terom@261:             }) :
terom@263:                 yield user, group
terom@261: 
terom@261:     if info.get('owner') :
terom@261:             for user in options.ldap.users.filter({
terom@261:                 'cn': info['owner'],
terom@261:             }) :
terom@263:                 yield user, None
terom@263: 
terom@263: def process_host_owner (options, host, info) :
terom@263:     """
terom@263:         Return (owner, comment) for host based on info, or None.
terom@263:     """
terom@263: 
terom@275:     owner = info.get('owner')
terom@275: 
terom@275:     if owner and owner.lower() in NONE_OWNERS :
terom@264:         return False
terom@263:     
terom@263:     # from ldap?
terom@263:     for ldap in process_host_owner_ldap(options, host, info) :
terom@263:         user, group = ldap
terom@263:         
terom@263:         if not group :
terom@263:             # get group from ldap
terom@263:             group = options.ldap.users.group(user)
terom@263:         
terom@263:         return user['uid'], u"{group} / {user}".format(
terom@263:                 user    = user.getunicode('cn'),
terom@263:                 group   = group.getunicode('cn'),
terom@263:         )
terom@261: 
terom@261: def process_host_comments (options, host, info) :
terom@261:     """
terom@261:         Process host fields from comment.
terom@261: 
terom@261:         Attempts to find owner from LDAP..
terom@261:     """
terom@261: 
terom@261:     log.debug("%s: %s", host, info)
terom@260:     
terom@263:     owner = process_host_owner(options, host, info) 
terom@261: 
terom@264:     if owner is False :
terom@264:         # do not mark any owner
terom@264:         pass
terom@264: 
terom@264:     elif owner :
terom@263:         owner, comment = owner
terom@263:         
terom@263:         log.info("%s: %s (%s)", host, owner, comment)
terom@263:         
terom@304:         yield 'comment.owner', comment
terom@261:         yield 'owner', owner,
terom@264: 
terom@279:     elif 'group' in info or 'owner' in info :
terom@264:         log.warn("%s: unknown owner: %s", host, info)
terom@304:         yield 'comment.owner', "{group} / {owner}".format(
terom@264:                 group   = info.get('group', ''),
terom@264:                 owner   = info.get('owner', ''),
terom@264:         )
terom@262:     
terom@262:     if info.get('host') :
terom@304:         yield 'comment.host', info['host']
terom@260: 
terom@260: def process_hosts_comments (options, import_hosts) :
terom@260:     """
terom@260:         Parse out comments from host imports..
terom@260:     """
terom@260: 
terom@260:     for host, field, value in import_hosts :
terom@260:         if field != 'comment':
terom@260:             yield host, field, value
terom@260:             continue
terom@260: 
terom@260:         fields = dict(process_zone_comment(options, host, value))
terom@261:         
terom@261:         if options.dump_host_comments :
terom@261:             print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(
terom@261:                     host        = host,
terom@261:                     comment     = value,
terom@261:                     group       = fields.get('group', ''),
terom@261:                     owner       = fields.get('owner', ''),
terom@261:                     mail        = fields.get('mail', ''),
terom@261:                     hostinfo    = fields.get('host', ''),
terom@261:             ).encode('utf-8')
terom@261:         
terom@260: 
terom@261:         for field, value in process_host_comments(options, host, fields) :
terom@261:             yield host, field, value
terom@263: 
terom@304: def import_hosts_files (options, zone_files, dhcp_files) :
terom@259:     """
terom@263:         Import host infos from given files.
terom@259:     """
terom@259: 
terom@304:     for zone_file in zone_files:
terom@301:         file = pvl.args.apply_file(zone_file, 'r', options.input_charset)
terom@301:         for info in import_zone_hosts(options, file) :
terom@263:             yield info
terom@263:     
terom@304:     for dhcp_file in dhcp_files :
terom@301:         file = pvl.args.apply_file(dhcp_file, 'r', options.input_charset)
terom@301:         for info in import_dhcp_conf(options, file) :
terom@263:             yield info
terom@263:        
terom@304: def process_import_hosts (options, import_hosts) :
terom@263:     """
terom@304:         Build hosts from imported fields.
terom@304: 
terom@304:         Yields (domain, host), { (field, ...): value }
terom@263:     """
terom@263:     
terom@263:     # gather
terom@259:     hosts = collections.defaultdict(lambda: collections.defaultdict(list))
terom@259: 
terom@304:     for (host, domain), field, value in import_hosts :
terom@304:         hosts[domain, host][tuple(field.split('.'))].append(value)
terom@259:     
terom@304:     # process
terom@304:     for (domain, host), fields in hosts.iteritems() :
terom@304:         SINGLE_FIELDS = (
terom@304:                 'ip',
terom@328:                 'ip.unused',
terom@304:                 'ip6',
terom@304:                 'comment.owner',
terom@304:                 'owner',
terom@304:                 'boot',
terom@304:         )
terom@304:         MULTI_FIELDS = (
terom@304:                 'comment.host',
terom@304:                 'ethernet',
terom@304:                 'alias',
terom@304:         ) 
terom@304:         host_fields = {}
terom@304: 
terom@304:         for field_name in SINGLE_FIELDS :
terom@304:             field = tuple(field_name.split('.'))
terom@304:             values = fields.get(field)
terom@304: 
terom@304:             if not values :
terom@304:                 continue
terom@304:             elif len(values) == 1 :
terom@304:                 value, = values
terom@304:             else :
terom@304:                 log.error("%s@%s: multiple %s: %s", host, domain, field, values)
terom@304:                 value = values[0]
terom@304:             
terom@304:             log.debug("%s@%s: %s: %s", host, domain, field, value)
terom@304:             host_fields[field] = value
terom@304:         
terom@304:         for field_name in MULTI_FIELDS :
terom@304:             field_prefix = tuple(field_name.split('.'))
terom@304:             
terom@304:             # find labled fields by prefix, or unlabled multi-fields
terom@304:             for field, values in fields.iteritems() :
terom@304:                 pre, field_index = field[:-1], field[-1]
terom@304:                 
terom@304:                 if not values :
terom@304:                     pass
terom@304: 
terom@304:                 elif pre == field_prefix :
terom@304:                     log.debug("%s@%s: %s.%s: %s", host, domain, field_prefix, field_index, value)
terom@304:                     host_fields[field] = values
terom@304: 
terom@304:                 elif field == field_prefix :
terom@304:                     log.debug("%s@%s: %s.*: %s", host, domain, field_prefix, value)
terom@304:                     host_fields[field_prefix] = values
terom@304: 
terom@318:         lazy_import = fields.get(tuple('lazy-import'.split('.')))
terom@318: 
terom@318:         if not lazy_import :
terom@318:             pass
terom@318:         elif set(host_fields) == set([('ip', )]) :
terom@318:             log.info("%s: omit lazy-import with fields: %s", host, ' '.join('.'.join(field) for field in host_fields))
terom@318:             continue
terom@318:         else :
terom@318:             log.info("%s: import lazy-import with fields: %s", host, ' '.join('.'.join(field) for field in host_fields))
terom@318: 
terom@304:         yield (host, domain), host_fields
terom@304: 
terom@304: def apply_import_hosts (options) :
terom@304:     """
terom@304:         Import hosts.
terom@304:     """
terom@304: 
terom@304:     import_hosts = import_hosts_files(options, options.import_zone_hosts, options.import_dhcp_hosts)
terom@304: 
terom@304:     # process
terom@304:     import_hosts = process_hosts_comments(options, import_hosts)
terom@304: 
terom@304:     # gather
terom@304:     return process_import_hosts(options, import_hosts)
terom@259: 
terom@313: def process_hosts (options, hosts) :
terom@313:     """
terom@313:         Sanity-check and post-process hosts.
terom@313: 
terom@313:         Does alias4 mapping, nonexistant alias checks, duplicate ip checks..
terom@313:     """
terom@313: 
terom@288:     by_name = dict(hosts)
terom@313:     by_ip = dict()
terom@288: 
terom@313:     # scan for alias4
terom@313:     for (host, domain), fields in by_name.items() :
terom@314:         for fmt, ip_field, alias_field in (
terom@314:                 (pvl.hosts.Host.ALIAS4_FMT, 'ip', 'alias4'),
terom@314:                 (pvl.hosts.Host.ALIAS6_FMT, 'ip6', 'alias6'),
terom@314:         ) :
terom@314:             alias = fmt.format(host=host)
terom@314:             alias_fields = by_name.get((alias, domain))
terom@313: 
terom@314:             if not alias_fields :
terom@314:                 continue
terom@313: 
terom@314:             elif alias_fields[(ip_field, )] != fields[(ip_field, )] :
terom@314:                 log.warn("%s: %s %s collision with %s", host, alias_field, ip_field, alias)
terom@314:             elif ('alias', ) in alias_fields :
terom@314:                 log.warn("%s: mapped to %s on %s", alias, alias_field, host)
terom@314:                 fields[(alias_field, )] = alias_fields.get(('alias', ), ())
terom@314:                 del by_name[(alias, domain)]
terom@314:             else :
terom@314:                 log.warn("%s: %s mapped to %s, but no aliases", alias, alias_field, host)
terom@315:                 del by_name[(alias, domain)]
terom@313: 
terom@320:     # scan by alias
terom@320:     by_alias = { }
terom@320: 
terom@320:     for (host, domain), fields in hosts :
terom@320:         for alias in fields.get(('alias', ), ()) :
terom@320:             if (alias, domain) in by_alias :
terom@320:                 log.warn("%s: duplicate alias %s: %s", host, alias, by_alias[(alias, domain)])
terom@320:             else :
terom@320:                 by_alias[(alias, domain)] = (host, fields)
terom@320: 
terom@313:     for (host, domain), fields in hosts :
terom@314:         fields = by_name.get((host, domain))
terom@314: 
terom@314:         if not fields :
terom@313:             # skip
terom@313:             continue
terom@313: 
terom@304:         if set(fields) == set([('alias', )]) :
terom@320:             aliases = fields[('alias', )]
terom@320:             if (host, domain) in by_alias :
terom@320:                 alias_host, alias_fields = by_alias[(host, domain)]
terom@320:                 log.info("%s: chain as alias to %s: %s", host, alias_host, ' '.join(aliases))
terom@320:                 alias_fields[('alias', )].extend(aliases)
terom@320:                 continue
terom@320:             else :
terom@320:                 log.warn("%s@%s: nonexistant alias target for: %s", host, domain, ' '.join(aliases))
terom@313:         
terom@313:         ip = fields.get(('ip', ))
terom@313: 
terom@313:         if ip in by_ip :
terom@313:             log.warn("%s: duplicate ip %s: %s", host, ip, by_ip[ip])
terom@328:         elif ip :
terom@328:             by_ip[ip] = host
terom@313:         else :
terom@328:             log.warn("%s: no ip", host)
terom@313: 
terom@313:         yield (host, domain), fields
terom@288: 
terom@288: def sort_export_hosts (options, hosts) :
terom@304:     """
terom@304:         Generate a sortable version of hosts, yielding (sort, host, fields).
terom@304:     """
terom@304: 
terom@265:     if options.output_prefix :
terom@265:         prefix = ipaddr.IPNetwork(options.output_prefix)
terom@265:     else :
terom@265:         prefix = None
terom@265: 
terom@304:     if options.output_domain :
terom@304:         select_domain = options.output_domain
terom@304:     else :
terom@304:         select_domain = None
terom@304: 
terom@304:     for (host, domain), fields in hosts :
terom@328:         ip = fields.get(('ip', )) or fields.get(('ip', 'unused'))
terom@304: 
terom@304:         log.debug("%s@%s: ip=%s", host, domain, ip)
terom@265:         
terom@265:         # sort by IP
terom@265:         if ip :
terom@304:             sort = ip
terom@265:         else :
terom@265:             # fake, to sort correctly
terom@265:             sort = ipaddr.IPAddress(0)
terom@265:         
terom@265:         # select
terom@327:         match = True
terom@327: 
terom@265:         if prefix:
terom@265:             if not (ip and ip in prefix) :
terom@327:                 match = False
terom@263: 
terom@304:         if select_domain :
terom@304:             if not (domain and domain == select_domain) :
terom@327:                 match = False
terom@304: 
terom@327:         if match and options.output_others :
terom@327:             pass
terom@327:         elif not match and not options.output_others :
terom@327:             pass
terom@327:         else :
terom@327:             yield (domain, sort), (host, domain), fields
terom@263: 
terom@263: def export_hosts (options, hosts) :
terom@263:     """
terom@285:         Generate hosts config lines for given hosts.
terom@263:     """
terom@263: 
terom@265:     # filter + sort
terom@288:     hosts = [(host, fields) for sort, host, fields in sorted(sort_export_hosts(options, hosts))]
terom@263: 
terom@304:     if options.output_domain :
terom@304:         # global
terom@304:         output_domain = False
terom@304:     else :
terom@304:         output_domain = None
terom@263: 
terom@304:     for (host, domain), fields in hosts :
terom@304:         if output_domain is False :
terom@304:             pass
terom@304:         elif domain != output_domain :
terom@304:             yield u"[{domain}]".format(domain=domain)
terom@304:             output_domain = domain
terom@304: 
terom@328:         # special handling for "unused" hosts
terom@328:         if ('ip', 'unused') in fields :
terom@328:             yield u"{indent}# {unused} {ip}".format(
terom@328:                     indent  = '\t' if output_domain else '',
terom@328:                     unused  = options.zone_unused,
terom@328:                     ip      = fields[('ip', 'unused')],
terom@328:             )
terom@328:             yield u""
terom@328:             continue
terom@328: 
terom@304:         # optional host-comments
terom@304:         for comment in fields.get(('comment', 'host'), ()):
terom@304:             yield u"{indent}# {comment}".format(
terom@304:                     indent  = '\t' if output_domain else '',
terom@304:                     comment = comment,
terom@304:             )
terom@304:         
terom@304:         if output_domain :
terom@304:             yield u"\t[[{host}]]".format(host=host)
terom@304:         else :
terom@304:             yield u"[{host}]".format(host=host)
terom@304: 
terom@304:         #if not options.output_domain and domain :
terom@304:         #    yield u"\t{field:15} = {domain}".format(field='domain', domain=domain)
terom@304:         
terom@304:         for field_name in (
terom@304:                 'ip',
terom@304:                 'ip6',
terom@304:                 'ethernet',
terom@304:                 'owner',
terom@304:                 'alias',
terom@314:                 'alias4',
terom@314:                 'alias6',
terom@304:                 'boot',
terom@304:         ) :
terom@304:             for field, value in fields.iteritems() :
terom@304:                 if field[0] == field_name :
terom@304:                     # optional field-comment
terom@304:                     comment = fields.get(('comment', field_name), None)
terom@304: 
terom@304:                     if isinstance(value, list) :
terom@304:                         value = ' '.join(value)
terom@304: 
terom@304:                     yield u"{indent}{field:15} = {value} {comment}".format(
terom@304:                             indent  = '\t\t' if output_domain else '\t',
terom@304:                             field   = '.'.join(str(label) for label in field),
terom@268:                             value   = value,
terom@304:                             comment = u"# {comment}".format(comment=comment) if comment else '',
terom@304:                     ).rstrip()
terom@285:     
terom@285:         yield ""
terom@285:    
terom@285: def apply_hosts_export (options, hosts) :
terom@285:     """
terom@285:         Export hosts to file.
terom@285:     """
terom@285: 
terom@285:     file = pvl.args.apply_file(options.output_hosts, 'w', options.output_charset)
terom@285: 
terom@285:     for line in export_hosts(options, hosts) :
terom@285:         print >>file, line
terom@259: 
terom@259: def main (argv) :
terom@259:     options, args = parse_options(argv)
terom@261: 
terom@261:     options.ldap = pvl.ldap.args.apply(options)
terom@259:     
terom@304:     # import
terom@304:     hosts = list(apply_import_hosts(options))
terom@259:    
terom@288:     # verify
terom@313:     hosts = process_hosts(options, hosts)
terom@288: 
terom@259:     # output
terom@259:     if options.output_hosts :
terom@285:         apply_hosts_export(options, hosts)
terom@259: 
terom@259: if __name__ == '__main__':
terom@259:     pvl.args.main(main)