1

#!/usr/bin/env python

     2

     3

"""

     4

    Import hosts from existing BIND or dhcpd files.

     5

"""

     6

     7

import pvl.args

     8

import pvl.dns.zone

     9

import pvl.dhcp.config

    10

import pvl.ldap.args

    11

    12

import ipaddr

    13

import optparse

    14

import collections

    15

import re

    16

import logging; log = logging.getLogger('pvl.hosts-import')

    17

    18

__version__ = '0.1'

    19

    20

def parse_options (argv) :

    21

    """

    22

        Parse command-line arguments.

    23

    """

    24

    25

    parser = optparse.OptionParser(

    26

            prog        = argv[0],

    27

            usage       = '%prog: [options]',

    28

            version     = __version__,

    29

    30

            # module docstring

    31

            description = __doc__,

    32

    )

    33

    34

    # logging

    35

    parser.add_option_group(pvl.args.parser(parser))

    36

    parser.add_option_group(pvl.ldap.args.parser(parser))

    37

    38

    parser.add_option('-c', '--input-charset',  metavar='CHARSET',  default='utf-8', 

    39

            help="Encoding used for input files")

    40

    41

    parser.add_option('--output-charset',       metavar='CHARSET',  default='utf-8', 

    42

            help="Encoding used for output files")

    43

    44

    # input

    45

    parser.add_option('--import-zone-hosts',    metavar='FILE',

    46

            help="Load hosts from DNS zone")

    47

    48

    parser.add_option('--import-dhcp-hosts',    metavar='FILE',

    49

            help="Load hosts from DHCP config")

    50

    51

    parser.add_option('--zone-comments-default-owner',  action='store_const',

    52

            dest='zone_comments_default', const='owner',

    53

            help="Import DNS zone comment as owner comment")

    54

    55

    parser.add_option('--zone-comments-default-host',   action='store_const',

    56

            dest='zone_comments_default', const='host',

    57

            help="Import DNS zone comment as host comment")

    58

    59

    parser.add_option('--dump-host-comments',   action='store_true',

    60

            help="Dump out info on imported host comments")

    61

    62

    # defaults

    63

    parser.add_option('--hosts-domain',         metavar='DOMAIN',

    64

            help="Default domain for hosts")

    65

    66

    parser.add_option('--zone-unused',          metavar='HOST',

    67

            help="DNS name for unallocated hosts")

    68

    69

    # output

    70

    parser.add_option('--output-hosts',         metavar='FILE',

    71

            help="Output hosts file")

    72

    73

    parser.add_option('--output-prefix',        metavar='PREFIX',

    74

            help="Select hosts by ip prefix")

    75

    76

    # defaults

    77

    parser.set_defaults(

    78

    79

    )

    80

    81

    # parse

    82

    options, args = parser.parse_args(argv[1:])

    83

    84

    # apply

    85

    pvl.args.apply(options, argv[0])

    86

    87

    return options, args

    88

    89

def import_zone_hosts (options, file) :

    90

    """

    91

        Yield host info from zonefile records.

    92

    """

    93

    98

        if options.zone_unused and rr.name == options.zone_unused :

    99

            log.debug("%s: skip %s", rr.name, rr)

   100

            continue

   101

   102

        elif rr.type in ('A', 'AAAA') :

   103

            ip, = rr.data

   104

   105

            yield rr.name, 'ip', ipaddr.IPAddress(ip)

   106

   107

            if rr.comment :

   108

                yield rr.name, 'comment', rr.comment

   109

   110

            if rr.origin :

   111

                yield rr.name, 'domain', rr.origin

   112

   113

        elif rr.type == 'CNAME' :

   114

            host, = rr.data

   115

   116

            yield host, 'alias', rr.name

   117

   118

        else :

   119

            log.warn("%s: unknown rr: %s", rr.name, rr)

   120

   121

def import_dhcp_host (options, host, items) :

   122

    """

   123

        Yield host infos from a dhcp host ... { ... }

   124

    """

   125

   126

    hostname = None

   127

    ethernet = []

   128

    fixed_address = None

   129

   130

    for item in items :

   131

        item, args = item[0], item[1:]

   132

   133

        if item == 'hardware' :

   134

            _ethernet, ethernet = args

   135

            assert _ethernet == 'ethernet'

   136

        elif item == 'fixed-address' :

   137

            fixed_address, = args

   138

        elif item == 'option' :

   139

            option = args.pop(0)

   140

   141

            if option == 'host-name' :

   142

                hostname, = args

   143

            else :

   144

                log.warn("host %s: ignore unknown option: %s", host, option)

   145

        else :

   146

            log.warn("host %s: ignore unknown item: %s", host, item)

   147

   148

    # determine hostname

   149

    suffix = None

   150

   151

    if '-' in host :

   152

        hostname, suffix = host.rsplit('-', 1)

   153

    else :

   154

        hostname = host

   155

   156

    if fixed_address and not re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :

   157

        hostname, domain = fixed_address.split('.', 1)

   158

   159

    #if suffix :

   160

    #    yield hostname, ('ethernet', suffix), ethernet

   161

    if hostname and ethernet :

   162

        yield hostname, 'ethernet', ethernet

   163

    else :

   164

        log.warn("%s: no hostname/ethernet: %s/%s", host, hostname, ethernet)

   165

   166

def import_dhcp_hosts (options, blocks) :

   167

    """

   168

        Process hosts from a parsed block

   169

    """

   170

   171

    for block, items, blocks in blocks :

   172

        log.info("%s", block)

   173

   174

        block, args = block[0], block[1:]

   175

   176

        if block == 'group' :

   177

            for info in import_dhcp_hosts(options, blocks) :

   178

                yield info

   179

        elif block == 'host' :

   180

            host, = args

   181

   182

            try :

   183

                for info in import_dhcp_host(options, host, items) :

   184

                    yield info

   185

            except ValueError as error :

   186

                log.warn("%s: invalid host: %s", host, error)

   187

        else:

   188

            log.warn("ignore unknown block: %s", block)

   189

   190

def import_dhcp_conf (options, file) :

   191

    items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)

   192

   193

    for item in items :

   194

        item, args = item[0], item[1:]

   195

   196

        if item == 'include' :

   197

            include, = args

   198

            for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :

   199

                yield info

   200

        else :

   201

            log.warn("ignore unknown item: %s", item)

   202

   203

    for info in import_dhcp_hosts(options, blocks) :

   204

        yield info

   205

   206

ZONE_COMMENTS = (

   207

        re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),

   208

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),

   209

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),

   210

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),

   211

)

   212

   213

ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')

   214

   215

def process_zone_comment (options, hostname, comment) :

   216

    """

   217

        Attempt to parse a host comment field... :D

   218

   219

        Yields (field, value) bits

   220

    """

   221

   222

    for regex in ZONE_COMMENTS :

   223

        match = regex.match(comment)

   224

   225

        if match :

   226

            matches = match.groupdict()

   229

            break

   230

    else :

   231

        if options.zone_comments_default :

   233

            matches = { options.zone_comments_default: comment }

   234

        else :

   235

            log.warn("%s: unknown comment: %s", hostname, comment)

   236

            return

   237

   238

    owner = matches.pop('owner', None)

   239

   240

    if owner :

   241

        mail_match = ZONE_OWNER_MAIL.match(owner)

   242

   243

        if mail_match :

   244

            mail_matches = mail_match.groupdict()

   245

   246

            owner = mail_matches['owner']

   247

            yield 'mail', mail_matches['mail'].strip()

   248

   249

        yield 'owner', owner.strip()

   250

   251

    for field, value in matches.iteritems() :

   252

        if value :

   253

            yield field, value.strip()

   254

   255

NONE_OWNERS = set((

   256

    u'tech',

   257

    u'atk',

   258

    u'toimisto',

   259

))

   260

   261

def process_host_owner_ldap (options, host, info) :

   262

    """

   263

        Yield guesses for user from LDAP.

   264

    """

   265

   266

    if info.get('mail') :

   267

        for user in options.ldap.users.filter(

   268

                { 'mailLocalAddress': info['mail'] },

   269

                { 'uid': info['mail'] },

   270

        ) :

   271

            yield user, None

   272

   273

    if info.get('group') and info.get('owner') :

   274

        groups = options.ldap.groups.filter(cn=info['group'])

   275

   276

        for group in groups :

   277

            for user in options.ldap.users.filter({

   278

                'gidNumber': group['gidNumber'],

   279

                'cn': info['owner'],

   280

            }) :

   281

                yield user, group

   282

   283

    if info.get('owner') :

   284

            for user in options.ldap.users.filter({

   285

                'cn': info['owner'],

   286

            }) :

   287

                yield user, None

   288

   289

def process_host_owner (options, host, info) :

   290

    """

   291

        Return (owner, comment) for host based on info, or None.

   292

    """

   293

   294

    owner = info.get('owner')

   295

   296

    if owner and owner.lower() in NONE_OWNERS :

   297

        return False

   298

   299

    # from ldap?

   300

    for ldap in process_host_owner_ldap(options, host, info) :

   301

        user, group = ldap

   302

   303

        if not group :

   304

            # get group from ldap

   305

            group = options.ldap.users.group(user)

   306

   307

        return user['uid'], u"{group} / {user}".format(

   308

                user    = user.getunicode('cn'),

   309

                group   = group.getunicode('cn'),

   310

        )

   311

   312

def process_host_comments (options, host, info) :

   313

    """

   314

        Process host fields from comment.

   315

   316

        Attempts to find owner from LDAP..

   317

    """

   318

   319

    log.debug("%s: %s", host, info)

   320

   321

    owner = process_host_owner(options, host, info) 

   322

   323

    if owner is False :

   324

        # do not mark any owner

   325

        pass

   326

   327

    elif owner :

   328

        owner, comment = owner

   329

   330

        log.info("%s: %s (%s)", host, owner, comment)

   331

   332

        yield 'comment-owner', comment

   333

        yield 'owner', owner,

   334

   336

        log.warn("%s: unknown owner: %s", host, info)

   337

        yield 'comment-owner', "{group} / {owner}".format(

   338

                group   = info.get('group', ''),

   339

                owner   = info.get('owner', ''),

   340

        )

   341

   342

    if info.get('host') :

   343

        yield 'comment-host', info['host']

   344

   345

def process_hosts_comments (options, import_hosts) :

   346

    """

   347

        Parse out comments from host imports..

   348

    """

   349

   350

    for host, field, value in import_hosts :

   351

        if field != 'comment':

   352

            yield host, field, value

   353

            continue

   354

   355

        fields = dict(process_zone_comment(options, host, value))

   356

   357

        if options.dump_host_comments :

   358

            print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(

   359

                    host        = host,

   360

                    comment     = value,

   361

                    group       = fields.get('group', ''),

   362

                    owner       = fields.get('owner', ''),

   363

                    mail        = fields.get('mail', ''),

   364

                    hostinfo    = fields.get('host', ''),

   365

            ).encode('utf-8')

   366

   367

   368

        for field, value in process_host_comments(options, host, fields) :

   369

            yield host, field, value

   370

   371

def apply_hosts_import (options) :

   372

    """

   373

        Import host infos from given files.

   374

    """

   375

   376

    if options.import_zone_hosts:

   377

        for info in import_zone_hosts(options,

   378

                pvl.args.apply_file(options.import_zone_hosts, 'r', options.input_charset)) :

   379

            yield info

   380

   381

    if options.import_dhcp_hosts:

   382

        for info in import_dhcp_conf(options,

   383

                pvl.args.apply_file(options.import_dhcp_hosts, 'r', options.input_charset)) :

   384

            yield info

   385

   386

def import_hosts (options) :

   387

    """

   388

        Import hosts from dns/dhcp.

   389

    """

   390

   391

    import_hosts = apply_hosts_import(options)

   392

    import_hosts = process_hosts_comments(options, import_hosts)

   393

   394

    # gather

   395

    hosts = collections.defaultdict(lambda: collections.defaultdict(list))

   396

   397

    for host, field, value in import_hosts :

   398

        hosts[host][field].append(value)

   399

   400

    return hosts.iteritems()

   401

   402

def process_export_hosts (options, hosts) :

   403

    if options.output_prefix :

   404

        prefix = ipaddr.IPNetwork(options.output_prefix)

   405

    else :

   406

        prefix = None

   407

   408

    for host, fields in hosts :

   409

        ip = fields.get('ip')

   410

   411

        # sort by IP

   412

        if ip :

   413

            sort = ip[0]

   414

        else :

   415

            # fake, to sort correctly

   416

            sort = ipaddr.IPAddress(0)

   417

   418

        # select

   419

        if prefix:

   420

            if not (ip and ip in prefix) :

   421

                continue

   422

   423

        yield sort, host, fields

   424

   425

def export_hosts (options, hosts) :

   426

    """

   427

        Export hosts to file.

   428

    """

   429

   430

    file = pvl.args.apply_file(options.output_hosts, 'w', options.output_charset)

   431

   432

    # filter + sort

   433

    hosts = [(host, fields) for sort, host, fields in sorted(process_export_hosts(options, hosts))]

   434

   435

    for host, fields in hosts :

   436

        for comment in fields.get('comment-host', ()):

   437

            print >>file, u"# {comment}".format(comment=comment)

   438

   439

        print >>file, u"[{host}]".format(host=host)

   440

   441

        for field, fmt in (

   442

                ('ip',              None),

   443

                ('ethernet',        None),

   444

                ('owner',           u"\t{field:15} = {value} # {fields[comment-owner][0]}"),

   445

                ('alias',           None),

   446

        ) :

   447

            if not fmt :

   448

                fmt = u"\t{field:15} = {value}"

   449

   450

            values = fields.get(field, ())

   451

   452

            if len(values) > 1 :

   453

                for index, value in enumerate(values, 1) :

   454

                    print >>file, fmt.format(

   455

                            field   = "{field}.{index}".format(field=field, index=index),

   456

                            value   = value,

   457

                            fields  = fields

   458

                    )

   459

            elif len(values) > 0 :

   460

                value, = values

   461

                print >>file, fmt.format(field=field, value=value, fields=fields)

   462

   463

        print >>file

   464

   465

def main (argv) :

   466

    options, args = parse_options(argv)

   467

   468

    options.ldap = pvl.ldap.args.apply(options)

   469

   470

    if args :

   471

        # direct from file

   472

        hosts = pvl.args.apply_files(args, 'r', options.input_charset)

   473

    else :

   474

        # import

   475

        hosts = import_hosts(options)

   476

   477

    # output

   478

    if options.output_hosts :

   479

        export_hosts(options, hosts)

   480

   481

if __name__ == '__main__':

   482

    pvl.args.main(main)