pvl-hosts: comparison bin/process-zone

equal deleted inserted replaced

-:d2e187c1f548
+:840092ee4d97
 parser.add_option('--reverse-domain',       metavar='DOMAIN',
 help="Domain to use for hosts in reverse zone")
 parser.add_option('--reverse-zone',         metavar='NET',
 help="Generate forward zone for given subnet (x.z.y)")
+#
+parser.add_option('--doctest',              action='store_true',
+help="Run module doctests")
 # defaults
 parser.set_defaults(
 loglevel            = logging.WARN,
 level   = options.loglevel,
 )
 return options, args
+class ZoneError (Exception) :
+pass
+class ZoneLineError (ZoneError) :
+"""
+ZoneLine-related error
+"""
+def __init__ (self, line, msg, *args, **kwargs) :
+super(ZoneLineError, self).__init__("%s: %s" % (line, msg.format(*args, **kwargs)))
 class ZoneLine (object) :
 """
 A line in a zonefile.
 """
 ts = None
 if line_timestamp_prefix :
 if ': ' not in line :
-raise Exception("Missing timestamp prefix on line: %s:%d: %s" % (file, lineno, line))
+raise ZoneError("%s:%d: Missing timestamp prefix: %s" % (file, lineno, line))
 # split prefix
 prefix, line = line.split(': ', 1)
 # parse it out
 """
 # the underlying line
 line = None
+# possible $ORIGIN context
+origin = None
 # record fields
 name = None
 type = None
 # list of data fields
 # optional
 ttl = None
 cls = None
 @classmethod
-def parse (cls, line) :
+def parse (cls, line, parts=None, origin=None) :
 """
 Parse from ZoneLine. Returns None if there is no record on the line..
 """
-if not line.parts :
+if parts is None :
+parts = list(line.parts)
+if not parts :
 # skip
 return
-# consume parts
-parts = list(line.parts)
 # indented lines don't have name
 if line.indent :
 name = None
 else :
 name = parts.pop(0)
-log.debug("  name=%r", name)
+log.debug("  name=%r, origin=%r", name, origin)
+if len(parts) < 2 :
+raise ZoneLineError(line, "Too few parts to parse: {0!r}", line.data)
 # parse ttl/cls/type
 ttl = _cls = None
 if parts and parts[0][0].isdigit() :
 data = parts
 log.debug("  ttl=%r, cls=%r, type=%r, data=%r", ttl, _cls, type, data)
 return cls(name, type, data,
+origin  = origin,
 ttl     = ttl,
 cls     = _cls,
 line    = line,
 )
-def __init__ (self, name, type, data, ttl=None, cls=None, line=None, comment=None) :
+def __init__ (self, name, type, data, origin=None, ttl=None, cls=None, line=None, comment=None) :
 self.name = name
 self.type = type
 self.data = data
 self.ttl = ttl
 self.cls = cls
+self.origin = origin
 self.line = line
 # XXX: within line
 self._comment = comment
 data    = ' '.join(unicode(data) for data in self.data),
 comment = comment,
 )
 def __str__ (self) :
-return ' '.join((self.name, self.type, ' '.join(self.data)))
+return ' '.join((self.name or '', self.type, ' '.join(self.data)))
 class TXTRecord (ZoneRecord) :
 """
 TXT record.
 """
 return super(TXTRecord, self).__init__(name, 'TXT',
 [u'"{0}"'.format(text.replace('"', '\\"'))],
 **opts
 )
-def parse_record (path, lineno, line, **opts) :
+class OffsetValue (object) :
-"""
+def __init__ (self, value) :
-Parse (name, ttl, type, data, comment) from bind zonefile.
+self.value = value
-Returns None for empty/comment lines.
+def __getitem__ (self, offset) :
-"""
+value = self.value + offset
-# line
+#log.debug("OffsetValue: %d[%d] -> %d", self.value, offset, value)
-line = ZoneLine.parse(path, lineno, line, **opts)
-record = ZoneRecord.parse(line)
+return value
-if record :
+def parse_generate_field (line, field) :
-return record
+"""
+Parse a $GENERATE lhs/rhs field:
-def parse_zone_records (file, **opts) :
+$
+${<offset>[,<width>[,<base>]]}
+\$
+$$
+Returns a wrapper that builds the field-value when called with the index.
+>>> parse_generate_field(None, "foo")(1)
+'foo'
+>>> parse_generate_field(None, "foo-$")(1)
+'foo-1'
+>>> parse_generate_field(None, "foo-$$")(1)
+'foo-$'
+>>> parse_generate_field(None, "\$")(1)
+'$'
+>>> parse_generate_field(None, "10.0.0.${100}")(1)
+'10.0.0.101'
+>>> parse_generate_field(None, "foo-${0,2,d}")(1)
+'foo-01'
+"""
+input = field
+expr = []
+while '$' in field :
+# defaults
+offset = 0
+width = 0
+base = 'd'
+escape = False
+# different forms
+if '${' in field :
+pre, body = field.split('${', 1)
+body, post = body.split('}', 1)
+# parse body
+parts = body.split(',')
+# offset
+offset = int(parts.pop(0))
+# width
+if parts :
+width = int(parts.pop(0))
+# base
+if parts :
+base = parts.pop(0)
+if parts:
+# fail
+raise ZoneLineError(line, "extra data in ${...} body: {0!r}", parts)
+elif '$$' in field :
+pre, post = field.split('$$', 1)
+escape = True
+elif '\\$' in field :
+pre, post = field.split('\\$', 1)
+escape = True
+else :
+pre, post = field.split('$', 1)
+expr.append(pre)
+if escape :
+expr.append('$')
+else :
+# meta-format
+fmt = '{value[%d]:0%d%s}' % (offset, width, base)
+log.debug("field=%r -> pre=%r, fmt=%r, post=%r", field, expr, fmt, post)
+expr.append(fmt)
+field = post
+# final
+if field :
+expr.append(field)
+# combine
+expr = ''.join(expr)
+log.debug("%s: %s", input, expr)
+# processed
+def value_func (value) :
+# magic wrapper to implement offsets
+return expr.format(value=OffsetValue(value))
+return value_func
+def process_generate (line, origin, parts) :
+"""
+Process a
+$GENERATE <start>-<stop>[/<step>] lhs [ttl] [class] type rhs [comment]
+directive into a series of ZoneResource's.
+"""
+range = parts.pop(0)
+# parse range
+if '/' in range :
+range, step = range.split('/')
+step = int(step)
+else :
+step = 1
+start, stop = range.split('-')
+start = int(start)
+stop = int(stop)
+log.debug("  range: start=%r, stop=%r, step=%r", start, stop, step)
+# inclusive
+range = xrange(start, stop + 1, step)
+lhs_func = parse_generate_field(line, parts.pop(0))
+rhs_func = parse_generate_field(line, parts.pop(-1))
+body = parts
+for i in range :
+# build
+parts = [lhs_func(i)] + body + [rhs_func(i)]
+log.debug(" %03d: %r", i, parts)
+# parse
+yield ZoneRecord.parse(line, parts=parts, origin=origin)
+def parse_zone_records (file, origin=None, **opts) :
 """
 Parse ZoneRecord items from the given zonefile, ignoring non-record lines.
 """
-for lineno, line in enumerate(file) :
+ttl = None
-record = parse_record(file.name, lineno, line, **opts)
+skip_multiline = False
+for lineno, raw_line in enumerate(file) :
+# parse comment
+if ';' in raw_line :
+line, comment = raw_line.split(';', 1)
+else :
+line = raw_line
+comment = None
+# XXX: handle multi-line statements...
+# start
+if '(' in line :
+skip_multiline = True
+log.warn("%s:%d: Start of multi-line statement: %s", file.name, lineno, raw_line)
+# end?
+if ')' in line :
+skip_multiline = False
+log.warn("%s:%d: End of multi-line statement: %s", file.name, lineno, raw_line)
+continue
+elif skip_multiline :
+log.warn("%s:%d: Multi-line statement: %s", file.name, lineno, raw_line)
+continue
+# parse
+line = ZoneLine.parse(file.name, lineno, raw_line, **opts)
+if not line.data :
+log.debug("%s: skip empty line: %s", line, raw_line)
+continue
+elif line.data.startswith('$') :
+# control record
+type = line.parts[0]
+if type == '$ORIGIN':
+# update
+origin = line.parts[1]
+log.info("%s: origin: %s", line, origin)
+elif type == '$GENERATE':
+# process...
+log.info("%s: generate: %s", line, line.parts)
+for record in process_generate(line, origin, line.parts[1:]) :
+yield record
+else :
+log.warning("%s: skip control record: %s", line, line.data)
+# XXX: passthrough!
+continue
+# normal record?
+record = ZoneRecord.parse(line, origin=origin)
 if record :
 yield record
-def check_zone_hosts (zone, whitelist=None) :
+else :
+# unknown
+log.warning("%s: skip unknown line: %s", line, line.data)
+def check_zone_hosts (zone, whitelist=None, whitelist_types=set(['TXT'])) :
 """
 Parse host/IP pairs from the zone, and verify that they are unique.
 As an exception, names listed in the given whitelist may have multiple IPs.
 """
 by_name = {}
 by_ip = {}
 fail = None
+last_name = None
 for r in zone :
-name = r.name
+name = r.name or last_name
+name = (r.origin, name)
 # name
-if name not in by_name :
+if r.type not in whitelist_types :
-by_name[name] = r
+if name not in by_name :
+by_name[name] = r
-elif r.name in whitelist :
-log.debug("Duplicate whitelist entry: %s", r)
+elif r.name in whitelist :
+log.debug("Duplicate whitelist entry: %s", r)
-else :
-# fail!
+else :
-log.warn("%s: Duplicate name: %s <-> %s", r.line, r, by_name[name])
+# fail!
-fail = True
+log.warn("%s: Duplicate name: %s <-> %s", r.line, r, by_name[name])
+fail = True
 # ip
 if r.type == 'A' :
 ip, = r.data
 assert 0 <= octet <= 255
 return '.'.join([str(octet) for octet in reversed(octets)] + ['in-addr', 'arpa'])
 def fqdn (*parts) :
-return '.'.join(parts) + '.'
+fqdn = '.'.join(parts)
+# we may be given an fqdn in parts
+if not fqdn.endswith('.') :
+fqdn += '.'
+return fqdn
 def process_zone_reverse (zone, origin, domain) :
 """
 Process zone data -> reverse zone data.
 """
 else :
 log.warning("Reverse does not match zone origin, skipping: (%s) -> %s <-> %s", ip, reverse, origin)
 continue
 # domain to use
-host_domain = domain
+host_domain = r.origin or domain
-host_fqdn = fqdn(r.name, domain)
+host_fqdn = fqdn(r.name, host_domain)
 yield ZoneRecord(reverse, 'PTR', [host_fqdn])
 def write_zone_records (file, zone) :
 for r in zone :
 def main (argv) :
 global options
 options, args = parse_options(argv)
+if options.doctest :
+import doctest
+fail, total = doctest.testmod()
+return fail
 if args :
 # open files
 input_files = [open_file(path, 'r', options.input_charset) for path in args]
 else :

changeset 558	840092ee4d97
parent 548	3d35d0eef197
child 603	b58236f9ea7b