1

#!/usr/bin/env python

     2

     3

"""

     4

    Import hosts from existing BIND or dhcpd files.

     5

"""

     6

     7

import pvl.args

     8

import pvl.dns.zone

     9

import pvl.dhcp.config

    10

import pvl.ldap.args

    11

    12

import ipaddr

    13

import optparse

    14

import collections

    15

import re

    16

import logging; log = logging.getLogger('pvl.hosts-import')

    17

    18

__version__ = '0.1'

    19

    20

def parse_options (argv) :

    21

    """

    22

        Parse command-line arguments.

    23

    """

    24

    25

    parser = optparse.OptionParser(

    26

            prog        = argv[0],

    27

            usage       = '%prog: [options]',

    28

            version     = __version__,

    29

    30

            # module docstring

    31

            description = __doc__,

    32

    )

    33

    34

    # logging

    35

    parser.add_option_group(pvl.args.parser(parser))

    36

    parser.add_option_group(pvl.ldap.args.parser(parser))

    37

    38

    parser.add_option('-c', '--input-charset',  metavar='CHARSET',  default='utf-8', 

    39

            help="Encoding used for input files")

    40

    41

    parser.add_option('--output-charset',       metavar='CHARSET',  default='utf-8', 

    42

            help="Encoding used for output files")

    43

    44

    # input

    45

    parser.add_option('--import-zone-hosts',    metavar='FILE',

    46

            help="Load hosts from DNS zone")

    47

    48

    parser.add_option('--import-dhcp-hosts',    metavar='FILE',

    49

            help="Load hosts from DHCP config")

    50

    51

    parser.add_option('--import-dhcp-boot-server',      metavar='NEXT-SERVER',

    52

            help="Default boot_server for dpc hosts")

    53

    54

    parser.add_option('--zone-comments-default-owner',  action='store_const',

    55

            dest='zone_comments_default', const='owner',

    56

            help="Import DNS zone comment as owner comment")

    57

    58

    parser.add_option('--zone-comments-default-host',   action='store_const',

    59

            dest='zone_comments_default', const='host',

    60

            help="Import DNS zone comment as host comment")

    61

    62

    parser.add_option('--dump-host-comments',   action='store_true',

    63

            help="Dump out info on imported host comments")

    64

    65

    # defaults

    66

    parser.add_option('--hosts-domain',         metavar='DOMAIN',

    67

            help="Default domain for hosts")

    68

    69

    parser.add_option('--zone-unused',          metavar='HOST',

    70

            help="DNS name for unallocated hosts")

    71

    72

    # output

    73

    parser.add_option('--output-hosts',         metavar='FILE',

    74

            help="Output hosts file")

    75

    76

    parser.add_option('--output-prefix',        metavar='PREFIX',

    77

            help="Select hosts by ip prefix")

    78

    79

    # defaults

    80

    parser.set_defaults(

    81

    82

    )

    83

    84

    # parse

    85

    options, args = parser.parse_args(argv[1:])

    86

    87

    # apply

    88

    pvl.args.apply(options, argv[0])

    89

    90

    return options, args

    91

    92

def import_zone_hosts (options, file) :

    93

    """

    94

        Yield host info from zonefile records.

    95

    """

    96

    97

    for rr in pvl.dns.zone.ZoneRecord.load(file,

    98

            # generated hosts need to imported by hand...

    99

            expand_generate = False,

   100

    ) :

   101

        if options.zone_unused and rr.name == options.zone_unused :

   102

            log.debug("%s: skip %s", rr.name, rr)

   103

            continue

   104

   105

        elif rr.type in ('A', 'AAAA') :

   106

            ip, = rr.data

   107

   108

            type = { 'A': 'ip', 'AAAA': 'ip6' }[rr.type]

   109

   110

            yield rr.name, type, ipaddr.IPAddress(ip)

   111

   112

            if rr.comment :

   113

                yield rr.name, 'comment', rr.comment

   114

   115

            if rr.origin :

   116

                # not a fqdn

   117

                yield rr.name, 'domain', rr.origin.rstrip('.')

   118

   119

        elif rr.type == 'CNAME' :

   120

            host, = rr.data

   121

   122

            yield host, 'alias', rr.name

   123

   124

        else :

   125

            log.warn("%s: unknown rr: %s", rr.name, rr)

   126

   127

def import_dhcp_host (options, host, items) :

   128

    """

   129

        Yield host infos from a dhcp host ... { ... }

   130

    """

   131

   132

    hostname = None

   133

    ethernet = []

   134

    fixed_address = None

   135

   136

    boot_server = options.import_dhcp_boot_server

   137

    boot_filename = None

   138

   139

    for item in items :

   140

        item, args = item[0], item[1:]

   141

   142

        if item == 'hardware' :

   143

            _ethernet, ethernet = args

   144

            assert _ethernet == 'ethernet'

   145

        elif item == 'fixed-address' :

   146

            fixed_address, = args

   147

        elif item == 'option' :

   148

            option = args.pop(0)

   149

   150

            if option == 'host-name' :

   151

                hostname, = args

   152

            else :

   153

                log.warn("host %s: ignore unknown option: %s", host, option)

   154

        elif item == 'next-sever' :

   155

            boot_server, = args

   156

        elif item == 'filename' :

   157

            boot_filename, = args

   158

        else :

   159

            log.warn("host %s: ignore unknown item: %s", host, item)

   160

   161

    # determine hostname

   162

    suffix = None

   163

   164

    if '-' in host :

   165

        hostname, suffix = host.rsplit('-', 1)

   166

    else :

   167

        hostname = host

   168

   169

    if fixed_address and not re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :

   170

        hostname, domain = fixed_address.split('.', 1)

   171

   172

    if not (hostname or ethernet) :

   173

        log.warn("%s: no hostname/ethernet: %s/%s", host, hostname, ethernet)

   174

        return

   175

   176

    yield hostname, 'ethernet', ethernet

   177

    #if suffix :

   178

    #    yield hostname, ('ethernet', suffix), ethernet

   179

   180

    if boot_server and boot_filename :

   181

        yield hostname, 'boot', "{server}:{filename}".format(

   182

                server      = boot_server,

   183

                filename    = boot_filename,

   184

        )

   185

    elif boot_filename :

   186

        yield hostname, 'boot', "{filename}".format(filename=boot_filename)

   187

   188

def import_dhcp_hosts (options, blocks) :

   189

    """

   190

        Process hosts from a parsed block

   191

    """

   192

   193

    for block, items, blocks in blocks :

   194

        log.info("%s", block)

   195

   196

        block, args = block[0], block[1:]

   197

   198

        if block == 'group' :

   199

            for info in import_dhcp_hosts(options, blocks) :

   200

                yield info

   201

        elif block == 'host' :

   202

            host, = args

   203

   204

            try :

   205

                for info in import_dhcp_host(options, host, items) :

   206

                    yield info

   207

            except ValueError as error :

   208

                log.exception("%s: invalid host: %s", host, error)

   209

        else:

   210

            log.warn("ignore unknown block: %s", block)

   211

   212

def import_dhcp_conf (options, file) :

   213

    items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)

   214

   215

    for item in items :

   216

        item, args = item[0], item[1:]

   217

   218

        if item == 'include' :

   219

            include, = args

   220

            for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :

   221

                yield info

   222

        else :

   223

            log.warn("ignore unknown item: %s", item)

   224

   225

    for info in import_dhcp_hosts(options, blocks) :

   226

        yield info

   227

   228

ZONE_COMMENTS = (

   229

        re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),

   230

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),

   231

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),

   232

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),

   233

)

   234

   235

ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')

   236

   237

def process_zone_comment (options, hostname, comment) :

   238

    """

   239

        Attempt to parse a host comment field... :D

   240

   241

        Yields (field, value) bits

   242

    """

   243

   244

    for regex in ZONE_COMMENTS :

   245

        match = regex.match(comment)

   246

   247

        if match :

   248

            matches = match.groupdict()

   249

   250

            log.info("%s: matched comment: %s", hostname, comment)

   251

            break

   252

    else :

   253

        if options.zone_comments_default :

   254

            log.info("%s: default comment: %s", hostname, comment)

   255

            matches = { options.zone_comments_default: comment }

   256

        else :

   257

            log.warn("%s: unknown comment: %s", hostname, comment)

   258

            return

   259

   260

    owner = matches.pop('owner', None)

   261

   262

    if owner :

   263

        mail_match = ZONE_OWNER_MAIL.match(owner)

   264

   265

        if mail_match :

   266

            mail_matches = mail_match.groupdict()

   267

   268

            owner = mail_matches['owner']

   269

            yield 'mail', mail_matches['mail'].strip()

   270

   271

        yield 'owner', owner.strip()

   272

   273

    for field, value in matches.iteritems() :

   274

        if value :

   275

            yield field, value.strip()

   276

   277

NONE_OWNERS = set((

   278

    u'tech',

   279

    u'atk',

   280

    u'toimisto',

   281

))

   282

   283

def process_host_owner_ldap (options, host, info) :

   284

    """

   285

        Yield guesses for user from LDAP.

   286

    """

   287

   288

    if info.get('mail') :

   289

        for user in options.ldap.users.filter(

   290

                { 'mailLocalAddress': info['mail'] },

   291

                { 'uid': info['mail'] },

   292

        ) :

   293

            yield user, None

   294

   295

    if info.get('group') and info.get('owner') :

   296

        groups = options.ldap.groups.filter(cn=info['group'])

   297

   298

        for group in groups :

   299

            for user in options.ldap.users.filter({

   300

                'gidNumber': group['gidNumber'],

   301

                'cn': info['owner'],

   302

            }) :

   303

                yield user, group

   304

   305

    if info.get('owner') :

   306

            for user in options.ldap.users.filter({

   307

                'cn': info['owner'],

   308

            }) :

   309

                yield user, None

   310

   311

def process_host_owner (options, host, info) :

   312

    """

   313

        Return (owner, comment) for host based on info, or None.

   314

    """

   315

   316

    owner = info.get('owner')

   317

   318

    if owner and owner.lower() in NONE_OWNERS :

   319

        return False

   320

   321

    # from ldap?

   322

    for ldap in process_host_owner_ldap(options, host, info) :

   323

        user, group = ldap

   324

   325

        if not group :

   326

            # get group from ldap

   327

            group = options.ldap.users.group(user)

   328

   329

        return user['uid'], u"{group} / {user}".format(

   330

                user    = user.getunicode('cn'),

   331

                group   = group.getunicode('cn'),

   332

        )

   333

   334

def process_host_comments (options, host, info) :

   335

    """

   336

        Process host fields from comment.

   337

   338

        Attempts to find owner from LDAP..

   339

    """

   340

   341

    log.debug("%s: %s", host, info)

   342

   343

    owner = process_host_owner(options, host, info) 

   344

   345

    if owner is False :

   346

        # do not mark any owner

   347

        pass

   348

   349

    elif owner :

   350

        owner, comment = owner

   351

   352

        log.info("%s: %s (%s)", host, owner, comment)

   353

   354

        yield 'comment-owner', comment

   355

        yield 'owner', owner,

   356

   357

    elif 'group' in info or 'owner' in info :

   358

        log.warn("%s: unknown owner: %s", host, info)

   359

        yield 'comment-owner', "{group} / {owner}".format(

   360

                group   = info.get('group', ''),

   361

                owner   = info.get('owner', ''),

   362

        )

   363

   364

    if info.get('host') :

   365

        yield 'comment-host', info['host']

   366

   367

def process_hosts_comments (options, import_hosts) :

   368

    """

   369

        Parse out comments from host imports..

   370

    """

   371

   372

    for host, field, value in import_hosts :

   373

        if field != 'comment':

   374

            yield host, field, value

   375

            continue

   376

   377

        fields = dict(process_zone_comment(options, host, value))

   378

   379

        if options.dump_host_comments :

   380

            print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(

   381

                    host        = host,

   382

                    comment     = value,

   383

                    group       = fields.get('group', ''),

   384

                    owner       = fields.get('owner', ''),

   385

                    mail        = fields.get('mail', ''),

   386

                    hostinfo    = fields.get('host', ''),

   387

            ).encode('utf-8')

   388

   389

   390

        for field, value in process_host_comments(options, host, fields) :

   391

            yield host, field, value

   392

   393

def apply_hosts_import (options) :

   394

    """

   395

        Import host infos from given files.

   396

    """

   397

   398

    if options.import_zone_hosts:

   399

        for info in import_zone_hosts(options,

   400

                pvl.args.apply_file(options.import_zone_hosts, 'r', options.input_charset)) :

   401

            yield info

   402

   403

    if options.import_dhcp_hosts:

   404

        for info in import_dhcp_conf(options,

   405

                pvl.args.apply_file(options.import_dhcp_hosts, 'r', options.input_charset)) :

   406

            yield info

   407

   408

def import_hosts (options) :

   409

    """

   410

        Import hosts from dns/dhcp.

   411

    """

   412

   413

    import_hosts = apply_hosts_import(options)

   414

    import_hosts = process_hosts_comments(options, import_hosts)

   415

   416

    # gather

   417

    hosts = collections.defaultdict(lambda: collections.defaultdict(list))

   418

   419

    for host, field, value in import_hosts :

   420

        hosts[host][field].append(value)

   421

   422

    return hosts.iteritems()

   423

   432

    if options.output_prefix :

   433

        prefix = ipaddr.IPNetwork(options.output_prefix)

   434

    else :

   435

        prefix = None

   436

   437

    for host, fields in hosts :

   438

        ip = fields.get('ip')

   439

   440

        # sort by IP

   441

        if ip :

   442

            sort = ip[0]

   443

        else :

   444

            # fake, to sort correctly

   445

            sort = ipaddr.IPAddress(0)

   446

   447

        # select

   448

        if prefix:

   449

            if not (ip and ip in prefix) :

   450

                continue

   451

   452

        yield sort, host, fields

   453

   454

def export_hosts (options, hosts) :

   455

    """

   456

        Generate hosts config lines for given hosts.

   457

    """

   458

   459

    FMT = u"\t{field:15} = {value}"

   460

   461

    yield u"[{domain}]".format(domain=options.hosts_domain)

   462

   463

    # filter + sort

   465

   466

    for host, fields in hosts :

   467

        for comment in fields.get('comment-host', ()):

   468

            yield u"# {comment}".format(comment=comment)

   469

   470

        yield u"[[{host}]]".format(host=host)

   471

   472

        for domain in fields.get('domain', ()) :

   473

            if domain != options.hosts_domain :

   474

                yield FMT.format(field='domain', value=domain)

   475

   476

        for field, fmt in (

   477

                ('ip',              FMT),

   478

                ('ip6',             FMT),

   479

                ('ethernet',        FMT),

   480

                ('owner',           u"\t{field:15} = {value} # {fields[comment-owner][0]}"),

   481

                ('alias',           FMT),

   482

                ('boot',            FMT),

   483

        ) :

   484

            values = fields.get(field, ())

   485

   486

            if len(values) > 1 :

   487

                for index, value in enumerate(values, 1) :

   488

                    yield fmt.format(

   489

                            field   = "{field}.{index}".format(field=field, index=index),

   490

                            value   = value,

   491

                            fields  = fields

   492

                    )

   493

            elif len(values) > 0 :

   494

                value, = values

   495

                yield fmt.format(field=field, value=value, fields=fields)

   496

   497

        yield ""