--- a/bin/pvl.dns-hosts Mon Dec 16 17:11:17 2013 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,450 +0,0 @@
-#!/usr/bin/env python
-
-"""
- Manipulate host definitions for dns/dhcp.
-"""
-
-import pvl.args, optparse
-import pvl.dns.zone
-import pvl.dhcp.config
-import pvl.ldap.args
-
-import ipaddr
-import optparse
-import collections
-import re
-import logging; log = logging.getLogger('main')
-
-__version__ = '0.1'
-
-def parse_options (argv) :
- """
- Parse command-line arguments.
- """
-
- parser = optparse.OptionParser(
- prog = argv[0],
- usage = '%prog: [options]',
- version = __version__,
-
- # module docstring
- description = __doc__,
- )
-
- # logging
- parser.add_option_group(pvl.args.parser(parser))
- parser.add_option_group(pvl.ldap.args.parser(parser))
-
- parser.add_option('-c', '--input-charset', metavar='CHARSET', default='utf-8',
- help="Encoding used for input files")
-
- parser.add_option('--output-charset', metavar='CHARSET', default='utf-8',
- help="Encoding used for output files")
-
- # input
- parser.add_option('--import-zone-hosts', metavar='FILE',
- help="Load hosts from DNS zone")
-
- parser.add_option('--import-dhcp-hosts', metavar='FILE',
- help="Load hosts from DHCP config")
-
- parser.add_option('--dump-host-comments', action='store_true',
- help="Dump out info on imported host comments")
-
- # defaults
- parser.add_option('--hosts-domain', metavar='DOMAIN',
- help="Default domain for hosts")
-
- parser.add_option('--zone-unused', metavar='HOST',
- help="DNS name for unallocated hosts")
-
- # output
- parser.add_option('--output-hosts', metavar='FILE',
- help="Output hosts file")
-
- parser.add_option('--output-prefix', metavar='PREFIX',
- help="Select hosts by ip prefix")
-
- # defaults
- parser.set_defaults(
-
- )
-
- # parse
- options, args = parser.parse_args(argv[1:])
-
- # apply
- pvl.args.apply(options, argv[0])
-
- return options, args
-
-def import_zone_hosts (options, file) :
- """
- Yield host info from zonefile records.
- """
-
- for rr in pvl.dns.zone.ZoneRecord.load(file) :
- if options.zone_unused and rr.name == options.zone_unused :
- log.debug("%s: skip %s", rr.name, rr)
- continue
-
- elif rr.type == 'A' :
- ip, = rr.data
-
- yield rr.name, 'ip', ipaddr.IPAddress(ip)
-
- if rr.comment :
- yield rr.name, 'comment', rr.comment
-
- elif rr.type == 'CNAME' :
- host, = rr.data
-
- yield host, 'alias', rr.name
-
- else :
- log.warn("%s: unknown rr: %s", rr.name, rr)
-
-def import_dhcp_host (options, host, items) :
- """
- Yield host infos from a dhcp host ... { ... }
- """
-
- hostname = None
- ethernet = []
- fixed_address = None
-
- for item in items :
- item, args = item[0], item[1:]
-
- if item == 'hardware' :
- _ethernet, ethernet = args
- assert _ethernet == 'ethernet'
- elif item == 'fixed-address' :
- fixed_address, = args
- elif item == 'option' :
- option = args.pop(0)
-
- if option == 'host-name' :
- hostname, = args
- else :
- log.warn("host %s: ignore unknown option: %s", host, option)
- else :
- log.warn("host %s: ignore unknown item: %s", host, item)
-
- # determine hostname
- suffix = None
-
- if '-' in host :
- hostname, suffix = host.rsplit('-', 1)
- else :
- hostname = host
-
- if fixed_address and not re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :
- hostname, domain = fixed_address.split('.', 1)
-
- #if suffix :
- # yield hostname, 'ethernet:{suffix}'.format(suffix=suffix), ethernet
- if hostname and ethernet :
- yield hostname, 'ethernet', ethernet
- else :
- log.warn("%s: no hostname/ethernet: %s/%s", host, hostname, ethernet)
-
-def import_dhcp_hosts (options, blocks) :
- """
- Process hosts from a parsed block
- """
-
- for block, items, blocks in blocks :
- log.info("%s", block)
-
- block, args = block[0], block[1:]
-
- if block == 'group' :
- for info in import_dhcp_hosts(options, blocks) :
- yield info
- elif block == 'host' :
- host, = args
-
- try :
- for info in import_dhcp_host(options, host, items) :
- yield info
- except ValueError as error :
- log.warn("%s: invalid host: %s", host, error)
- else:
- log.warn("ignore unknown block: %s", block)
-
-def import_dhcp_conf (options, file) :
- items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)
-
- for item in items :
- item, args = item[0], item[1:]
-
- if item == 'include' :
- include, = args
- for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :
- yield info
- else :
- log.warn("ignore unknown item: %s", item)
-
- for info in import_dhcp_hosts(options, blocks) :
- yield info
-
-ZONE_COMMENTS = (
- re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),
- re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),
- re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),
- re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),
- re.compile(r'(?P<owner>.+)'),
-)
-
-ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')
-
-def process_zone_comment (options, hostname, comment) :
- """
- Attempt to parse a host comment field... :D
-
- Yields (field, value) bits
- """
-
- for regex in ZONE_COMMENTS :
- match = regex.match(comment)
-
- if match :
- break
- else :
- log.warn("%s: unparsed comment: %s", hostname, comment)
- return
-
- matches = match.groupdict()
- owner = matches.pop('owner', None)
-
- if owner :
- mail_match = ZONE_OWNER_MAIL.match(owner)
-
- if mail_match :
- mail_matches = mail_match.groupdict()
-
- owner = mail_matches['owner']
- yield 'mail', mail_matches['mail'].strip()
-
- yield 'owner', owner.strip()
-
- for field, value in matches.iteritems() :
- if value :
- yield field, value.strip()
-
-NONE_OWNERS = set((
- u'tech',
- u'atk',
- u'toimisto',
-))
-
-def process_host_owner_ldap (options, host, info) :
- """
- Yield guesses for user from LDAP.
- """
-
- if info.get('mail') :
- for user in options.ldap.users.filter(
- { 'mailLocalAddress': info['mail'] },
- { 'uid': info['mail'] },
- ) :
- yield user, None
-
- if info.get('group') and info.get('owner') :
- groups = options.ldap.groups.filter(cn=info['group'])
-
- for group in groups :
- for user in options.ldap.users.filter({
- 'gidNumber': group['gidNumber'],
- 'cn': info['owner'],
- }) :
- yield user, group
-
- if info.get('owner') :
- for user in options.ldap.users.filter({
- 'cn': info['owner'],
- }) :
- yield user, None
-
-def process_host_owner (options, host, info) :
- """
- Return (owner, comment) for host based on info, or None.
- """
-
- if info.get('owner').lower() in NONE_OWNERS :
- return False
-
- # from ldap?
- for ldap in process_host_owner_ldap(options, host, info) :
- user, group = ldap
-
- if not group :
- # get group from ldap
- group = options.ldap.users.group(user)
-
- return user['uid'], u"{group} / {user}".format(
- user = user.getunicode('cn'),
- group = group.getunicode('cn'),
- )
-
-def process_host_comments (options, host, info) :
- """
- Process host fields from comment.
-
- Attempts to find owner from LDAP..
- """
-
- log.debug("%s: %s", host, info)
-
- owner = process_host_owner(options, host, info)
-
- if owner is False :
- # do not mark any owner
- pass
-
- elif owner :
- owner, comment = owner
-
- log.info("%s: %s (%s)", host, owner, comment)
-
- yield 'owner-comment', comment
- yield 'owner', owner,
-
- else :
- log.warn("%s: unknown owner: %s", host, info)
- yield 'comment', "owner: {group} / {owner}".format(
- group = info.get('group', ''),
- owner = info.get('owner', ''),
- )
-
- if info.get('host') :
- yield 'comment', info['host']
-
-def process_hosts_comments (options, import_hosts) :
- """
- Parse out comments from host imports..
- """
-
- for host, field, value in import_hosts :
- if field != 'comment':
- yield host, field, value
- continue
-
- fields = dict(process_zone_comment(options, host, value))
-
- if options.dump_host_comments :
- print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(
- host = host,
- comment = value,
- group = fields.get('group', ''),
- owner = fields.get('owner', ''),
- mail = fields.get('mail', ''),
- hostinfo = fields.get('host', ''),
- ).encode('utf-8')
-
-
- for field, value in process_host_comments(options, host, fields) :
- yield host, field, value
-
-def apply_hosts_import (options) :
- """
- Import host infos from given files.
- """
-
- if options.import_zone_hosts:
- for info in import_zone_hosts(options,
- pvl.args.apply_file(options.import_zone_hosts)) :
- yield info
-
- if options.import_dhcp_hosts:
- for info in import_dhcp_conf(options,
- pvl.args.apply_file(options.import_dhcp_hosts)) :
- yield info
-
-def import_hosts (options) :
- """
- Import hosts from dns/dhcp.
- """
-
- import_hosts = apply_hosts_import(options)
- import_hosts = process_hosts_comments(options, import_hosts)
-
- # gather
- hosts = collections.defaultdict(lambda: collections.defaultdict(list))
-
- for host, field, value in import_hosts :
- hosts[host][field].append(value)
-
- return hosts.iteritems()
-
-def process_export_hosts (options, hosts) :
- if options.output_prefix :
- prefix = ipaddr.IPNetwork(options.output_prefix)
- else :
- prefix = None
-
- for host, fields in hosts :
- ip = fields.get('ip')
-
- # sort by IP
- if ip :
- sort = ip = ip[0]
- else :
- # fake, to sort correctly
- sort = ipaddr.IPAddress(0)
-
- # select
- if prefix:
- if not (ip and ip in prefix) :
- continue
-
- yield sort, host, fields
-
-def export_hosts (options, hosts) :
- """
- Export hosts to file.
- """
-
- file = pvl.args.apply_file(options.output_hosts, 'w', options.output_charset)
-
- # filter + sort
- hosts = [(host, fields) for sort, host, fields in sorted(process_export_hosts(options, hosts))]
-
- for host, fields in hosts :
- for comment in fields.get('comment', ()) :
- print >>file, u"# {comment}".format(comment=comment)
-
- print >>file, u"[{host}]".format(host=host)
-
- for field, fmt in (
- ('ip', None),
- ('ethernet', None),
- ('owner', u"\t{field:15} = {value} # {fields[owner-comment][0]}"),
- ) :
- if not fmt :
- fmt = u"\t{field:15} = {value}"
-
- for value in fields.get(field, ()) :
- print >>file, fmt.format(field=field, value=value, fields=fields)
-
- print >>file
-
-def main (argv) :
- options, args = parse_options(argv)
-
- options.ldap = pvl.ldap.args.apply(options)
-
- if args :
- # direct from file
- hosts = pvl.args.apply_files(args, 'r', options.input_charset)
- else :
- # import
- hosts = import_hosts(options)
-
- # output
- if options.output_hosts :
- export_hosts(options, hosts)
-
-if __name__ == '__main__':
- pvl.args.main(main)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/pvl.hosts-import Mon Dec 16 17:11:56 2013 +0200
@@ -0,0 +1,450 @@
+#!/usr/bin/env python
+
+"""
+ Manipulate host definitions for dns/dhcp.
+"""
+
+import pvl.args, optparse
+import pvl.dns.zone
+import pvl.dhcp.config
+import pvl.ldap.args
+
+import ipaddr
+import optparse
+import collections
+import re
+import logging; log = logging.getLogger('pvl.hosts-import')
+
+__version__ = '0.1'
+
+def parse_options (argv) :
+ """
+ Parse command-line arguments.
+ """
+
+ parser = optparse.OptionParser(
+ prog = argv[0],
+ usage = '%prog: [options]',
+ version = __version__,
+
+ # module docstring
+ description = __doc__,
+ )
+
+ # logging
+ parser.add_option_group(pvl.args.parser(parser))
+ parser.add_option_group(pvl.ldap.args.parser(parser))
+
+ parser.add_option('-c', '--input-charset', metavar='CHARSET', default='utf-8',
+ help="Encoding used for input files")
+
+ parser.add_option('--output-charset', metavar='CHARSET', default='utf-8',
+ help="Encoding used for output files")
+
+ # input
+ parser.add_option('--import-zone-hosts', metavar='FILE',
+ help="Load hosts from DNS zone")
+
+ parser.add_option('--import-dhcp-hosts', metavar='FILE',
+ help="Load hosts from DHCP config")
+
+ parser.add_option('--dump-host-comments', action='store_true',
+ help="Dump out info on imported host comments")
+
+ # defaults
+ parser.add_option('--hosts-domain', metavar='DOMAIN',
+ help="Default domain for hosts")
+
+ parser.add_option('--zone-unused', metavar='HOST',
+ help="DNS name for unallocated hosts")
+
+ # output
+ parser.add_option('--output-hosts', metavar='FILE',
+ help="Output hosts file")
+
+ parser.add_option('--output-prefix', metavar='PREFIX',
+ help="Select hosts by ip prefix")
+
+ # defaults
+ parser.set_defaults(
+
+ )
+
+ # parse
+ options, args = parser.parse_args(argv[1:])
+
+ # apply
+ pvl.args.apply(options, argv[0])
+
+ return options, args
+
+def import_zone_hosts (options, file) :
+ """
+ Yield host info from zonefile records.
+ """
+
+ for rr in pvl.dns.zone.ZoneRecord.load(file) :
+ if options.zone_unused and rr.name == options.zone_unused :
+ log.debug("%s: skip %s", rr.name, rr)
+ continue
+
+ elif rr.type == 'A' :
+ ip, = rr.data
+
+ yield rr.name, 'ip', ipaddr.IPAddress(ip)
+
+ if rr.comment :
+ yield rr.name, 'comment', rr.comment
+
+ elif rr.type == 'CNAME' :
+ host, = rr.data
+
+ yield host, 'alias', rr.name
+
+ else :
+ log.warn("%s: unknown rr: %s", rr.name, rr)
+
+def import_dhcp_host (options, host, items) :
+ """
+ Yield host infos from a dhcp host ... { ... }
+ """
+
+ hostname = None
+ ethernet = []
+ fixed_address = None
+
+ for item in items :
+ item, args = item[0], item[1:]
+
+ if item == 'hardware' :
+ _ethernet, ethernet = args
+ assert _ethernet == 'ethernet'
+ elif item == 'fixed-address' :
+ fixed_address, = args
+ elif item == 'option' :
+ option = args.pop(0)
+
+ if option == 'host-name' :
+ hostname, = args
+ else :
+ log.warn("host %s: ignore unknown option: %s", host, option)
+ else :
+ log.warn("host %s: ignore unknown item: %s", host, item)
+
+ # determine hostname
+ suffix = None
+
+ if '-' in host :
+ hostname, suffix = host.rsplit('-', 1)
+ else :
+ hostname = host
+
+ if fixed_address and not re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :
+ hostname, domain = fixed_address.split('.', 1)
+
+ #if suffix :
+ # yield hostname, 'ethernet:{suffix}'.format(suffix=suffix), ethernet
+ if hostname and ethernet :
+ yield hostname, 'ethernet', ethernet
+ else :
+ log.warn("%s: no hostname/ethernet: %s/%s", host, hostname, ethernet)
+
+def import_dhcp_hosts (options, blocks) :
+ """
+ Process hosts from a parsed block
+ """
+
+ for block, items, blocks in blocks :
+ log.info("%s", block)
+
+ block, args = block[0], block[1:]
+
+ if block == 'group' :
+ for info in import_dhcp_hosts(options, blocks) :
+ yield info
+ elif block == 'host' :
+ host, = args
+
+ try :
+ for info in import_dhcp_host(options, host, items) :
+ yield info
+ except ValueError as error :
+ log.warn("%s: invalid host: %s", host, error)
+ else:
+ log.warn("ignore unknown block: %s", block)
+
+def import_dhcp_conf (options, file) :
+ items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)
+
+ for item in items :
+ item, args = item[0], item[1:]
+
+ if item == 'include' :
+ include, = args
+ for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :
+ yield info
+ else :
+ log.warn("ignore unknown item: %s", item)
+
+ for info in import_dhcp_hosts(options, blocks) :
+ yield info
+
+ZONE_COMMENTS = (
+ re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),
+ re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),
+ re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),
+ re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),
+ re.compile(r'(?P<owner>.+)'),
+)
+
+ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')
+
+def process_zone_comment (options, hostname, comment) :
+ """
+ Attempt to parse a host comment field... :D
+
+ Yields (field, value) bits
+ """
+
+ for regex in ZONE_COMMENTS :
+ match = regex.match(comment)
+
+ if match :
+ break
+ else :
+ log.warn("%s: unparsed comment: %s", hostname, comment)
+ return
+
+ matches = match.groupdict()
+ owner = matches.pop('owner', None)
+
+ if owner :
+ mail_match = ZONE_OWNER_MAIL.match(owner)
+
+ if mail_match :
+ mail_matches = mail_match.groupdict()
+
+ owner = mail_matches['owner']
+ yield 'mail', mail_matches['mail'].strip()
+
+ yield 'owner', owner.strip()
+
+ for field, value in matches.iteritems() :
+ if value :
+ yield field, value.strip()
+
+NONE_OWNERS = set((
+ u'tech',
+ u'atk',
+ u'toimisto',
+))
+
+def process_host_owner_ldap (options, host, info) :
+ """
+ Yield guesses for user from LDAP.
+ """
+
+ if info.get('mail') :
+ for user in options.ldap.users.filter(
+ { 'mailLocalAddress': info['mail'] },
+ { 'uid': info['mail'] },
+ ) :
+ yield user, None
+
+ if info.get('group') and info.get('owner') :
+ groups = options.ldap.groups.filter(cn=info['group'])
+
+ for group in groups :
+ for user in options.ldap.users.filter({
+ 'gidNumber': group['gidNumber'],
+ 'cn': info['owner'],
+ }) :
+ yield user, group
+
+ if info.get('owner') :
+ for user in options.ldap.users.filter({
+ 'cn': info['owner'],
+ }) :
+ yield user, None
+
+def process_host_owner (options, host, info) :
+ """
+ Return (owner, comment) for host based on info, or None.
+ """
+
+ if info.get('owner').lower() in NONE_OWNERS :
+ return False
+
+ # from ldap?
+ for ldap in process_host_owner_ldap(options, host, info) :
+ user, group = ldap
+
+ if not group :
+ # get group from ldap
+ group = options.ldap.users.group(user)
+
+ return user['uid'], u"{group} / {user}".format(
+ user = user.getunicode('cn'),
+ group = group.getunicode('cn'),
+ )
+
+def process_host_comments (options, host, info) :
+ """
+ Process host fields from comment.
+
+ Attempts to find owner from LDAP..
+ """
+
+ log.debug("%s: %s", host, info)
+
+ owner = process_host_owner(options, host, info)
+
+ if owner is False :
+ # do not mark any owner
+ pass
+
+ elif owner :
+ owner, comment = owner
+
+ log.info("%s: %s (%s)", host, owner, comment)
+
+ yield 'owner-comment', comment
+ yield 'owner', owner,
+
+ else :
+ log.warn("%s: unknown owner: %s", host, info)
+ yield 'comment', "owner: {group} / {owner}".format(
+ group = info.get('group', ''),
+ owner = info.get('owner', ''),
+ )
+
+ if info.get('host') :
+ yield 'comment', info['host']
+
+def process_hosts_comments (options, import_hosts) :
+ """
+ Parse out comments from host imports..
+ """
+
+ for host, field, value in import_hosts :
+ if field != 'comment':
+ yield host, field, value
+ continue
+
+ fields = dict(process_zone_comment(options, host, value))
+
+ if options.dump_host_comments :
+ print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(
+ host = host,
+ comment = value,
+ group = fields.get('group', ''),
+ owner = fields.get('owner', ''),
+ mail = fields.get('mail', ''),
+ hostinfo = fields.get('host', ''),
+ ).encode('utf-8')
+
+
+ for field, value in process_host_comments(options, host, fields) :
+ yield host, field, value
+
+def apply_hosts_import (options) :
+ """
+ Import host infos from given files.
+ """
+
+ if options.import_zone_hosts:
+ for info in import_zone_hosts(options,
+ pvl.args.apply_file(options.import_zone_hosts)) :
+ yield info
+
+ if options.import_dhcp_hosts:
+ for info in import_dhcp_conf(options,
+ pvl.args.apply_file(options.import_dhcp_hosts)) :
+ yield info
+
+def import_hosts (options) :
+ """
+ Import hosts from dns/dhcp.
+ """
+
+ import_hosts = apply_hosts_import(options)
+ import_hosts = process_hosts_comments(options, import_hosts)
+
+ # gather
+ hosts = collections.defaultdict(lambda: collections.defaultdict(list))
+
+ for host, field, value in import_hosts :
+ hosts[host][field].append(value)
+
+ return hosts.iteritems()
+
+def process_export_hosts (options, hosts) :
+ if options.output_prefix :
+ prefix = ipaddr.IPNetwork(options.output_prefix)
+ else :
+ prefix = None
+
+ for host, fields in hosts :
+ ip = fields.get('ip')
+
+ # sort by IP
+ if ip :
+ sort = ip = ip[0]
+ else :
+ # fake, to sort correctly
+ sort = ipaddr.IPAddress(0)
+
+ # select
+ if prefix:
+ if not (ip and ip in prefix) :
+ continue
+
+ yield sort, host, fields
+
+def export_hosts (options, hosts) :
+ """
+ Export hosts to file.
+ """
+
+ file = pvl.args.apply_file(options.output_hosts, 'w', options.output_charset)
+
+ # filter + sort
+ hosts = [(host, fields) for sort, host, fields in sorted(process_export_hosts(options, hosts))]
+
+ for host, fields in hosts :
+ for comment in fields.get('comment', ()) :
+ print >>file, u"# {comment}".format(comment=comment)
+
+ print >>file, u"[{host}]".format(host=host)
+
+ for field, fmt in (
+ ('ip', None),
+ ('ethernet', None),
+ ('owner', u"\t{field:15} = {value} # {fields[owner-comment][0]}"),
+ ) :
+ if not fmt :
+ fmt = u"\t{field:15} = {value}"
+
+ for value in fields.get(field, ()) :
+ print >>file, fmt.format(field=field, value=value, fields=fields)
+
+ print >>file
+
+def main (argv) :
+ options, args = parse_options(argv)
+
+ options.ldap = pvl.ldap.args.apply(options)
+
+ if args :
+ # direct from file
+ hosts = pvl.args.apply_files(args, 'r', options.input_charset)
+ else :
+ # import
+ hosts = import_hosts(options)
+
+ # output
+ if options.output_hosts :
+ export_hosts(options, hosts)
+
+if __name__ == '__main__':
+ pvl.args.main(main)