1

#!/usr/bin/env python

     2

     3

"""

     4

    Import hosts from existing BIND or dhcpd files.

     5

"""

     6

     7

import pvl.args

     8

import pvl.dns.zone

     9

import pvl.dhcp.config

    10

import pvl.ldap.args

    11

    12

import ipaddr

    13

import optparse

    14

import collections

    15

import re

    16

import logging; log = logging.getLogger('pvl.hosts-import')

    17

    18

__version__ = '0.1'

    19

    20

def parse_options (argv) :

    21

    """

    22

        Parse command-line arguments.

    23

    """

    24

    25

    parser = optparse.OptionParser(

    26

            prog        = argv[0],

    27

            usage       = '%prog: [options]',

    28

            version     = __version__,

    29

    30

            # module docstring

    31

            description = __doc__,

    32

    )

    33

    34

    # logging

    35

    parser.add_option_group(pvl.args.parser(parser))

    36

    parser.add_option_group(pvl.ldap.args.parser(parser))

    37

    38

    parser.add_option('-c', '--input-charset',  metavar='CHARSET',  default='utf-8', 

    39

            help="Encoding used for input files")

    40

    41

    parser.add_option('--output-charset',       metavar='CHARSET',  default='utf-8', 

    42

            help="Encoding used for output files")

    43

    44

    # input

    45

    parser.add_option('--import-zone-hosts',    metavar='FILE',

    46

            help="Load hosts from DNS zone")

    47

    48

    parser.add_option('--import-dhcp-hosts',    metavar='FILE',

    49

            help="Load hosts from DHCP config")

    50

    59

    parser.add_option('--dump-host-comments',   action='store_true',

    60

            help="Dump out info on imported host comments")

    61

    62

    # defaults

    63

    parser.add_option('--hosts-domain',         metavar='DOMAIN',

    64

            help="Default domain for hosts")

    65

    66

    parser.add_option('--zone-unused',          metavar='HOST',

    67

            help="DNS name for unallocated hosts")

    68

    69

    # output

    70

    parser.add_option('--output-hosts',         metavar='FILE',

    71

            help="Output hosts file")

    72

    73

    parser.add_option('--output-prefix',        metavar='PREFIX',

    74

            help="Select hosts by ip prefix")

    75

    76

    # defaults

    77

    parser.set_defaults(

    78

    79

    )

    80

    81

    # parse

    82

    options, args = parser.parse_args(argv[1:])

    83

    84

    # apply

    85

    pvl.args.apply(options, argv[0])

    86

    87

    return options, args

    88

    89

def import_zone_hosts (options, file) :

    90

    """

    91

        Yield host info from zonefile records.

    92

    """

    93

    94

    for rr in pvl.dns.zone.ZoneRecord.load(file) :

    95

        if options.zone_unused and rr.name == options.zone_unused :

    96

            log.debug("%s: skip %s", rr.name, rr)

    97

            continue

    98

   100

            ip, = rr.data

   101

   102

            yield rr.name, 'ip', ipaddr.IPAddress(ip)

   103

   104

            if rr.comment :

   105

                yield rr.name, 'comment', rr.comment

   106

   110

        elif rr.type == 'CNAME' :

   111

            host, = rr.data

   112

   113

            yield host, 'alias', rr.name

   114

   115

        else :

   116

            log.warn("%s: unknown rr: %s", rr.name, rr)

   117

   118

def import_dhcp_host (options, host, items) :

   119

    """

   120

        Yield host infos from a dhcp host ... { ... }

   121

    """

   122

   123

    hostname = None

   124

    ethernet = []

   125

    fixed_address = None

   126

   127

    for item in items :

   128

        item, args = item[0], item[1:]

   129

   130

        if item == 'hardware' :

   131

            _ethernet, ethernet = args

   132

            assert _ethernet == 'ethernet'

   133

        elif item == 'fixed-address' :

   134

            fixed_address, = args

   135

        elif item == 'option' :

   136

            option = args.pop(0)

   137

   138

            if option == 'host-name' :

   139

                hostname, = args

   140

            else :

   141

                log.warn("host %s: ignore unknown option: %s", host, option)

   142

        else :

   143

            log.warn("host %s: ignore unknown item: %s", host, item)

   144

   145

    # determine hostname

   146

    suffix = None

   147

   148

    if '-' in host :

   149

        hostname, suffix = host.rsplit('-', 1)

   150

    else :

   151

        hostname = host

   152

   153

    if fixed_address and not re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :

   154

        hostname, domain = fixed_address.split('.', 1)

   155

   156

    #if suffix :

   157

    #    yield hostname, ('ethernet', suffix), ethernet

   158

    if hostname and ethernet :

   159

        yield hostname, 'ethernet', ethernet

   160

    else :

   161

        log.warn("%s: no hostname/ethernet: %s/%s", host, hostname, ethernet)

   162

   163

def import_dhcp_hosts (options, blocks) :

   164

    """

   165

        Process hosts from a parsed block

   166

    """

   167

   168

    for block, items, blocks in blocks :

   169

        log.info("%s", block)

   170

   171

        block, args = block[0], block[1:]

   172

   173

        if block == 'group' :

   174

            for info in import_dhcp_hosts(options, blocks) :

   175

                yield info

   176

        elif block == 'host' :

   177

            host, = args

   178

   179

            try :

   180

                for info in import_dhcp_host(options, host, items) :

   181

                    yield info

   182

            except ValueError as error :

   183

                log.warn("%s: invalid host: %s", host, error)

   184

        else:

   185

            log.warn("ignore unknown block: %s", block)

   186

   187

def import_dhcp_conf (options, file) :

   188

    items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)

   189

   190

    for item in items :

   191

        item, args = item[0], item[1:]

   192

   193

        if item == 'include' :

   194

            include, = args

   195

            for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :

   196

                yield info

   197

        else :

   198

            log.warn("ignore unknown item: %s", item)

   199

   200

    for info in import_dhcp_hosts(options, blocks) :

   201

        yield info

   202

   203

ZONE_COMMENTS = (

   204

        re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),

   205

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),

   206

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),

   207

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),

   208

)

   209

   210

ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')

   211

   212

def process_zone_comment (options, hostname, comment) :

   213

    """

   214

        Attempt to parse a host comment field... :D

   215

   216

        Yields (field, value) bits

   217

    """

   218

   219

    for regex in ZONE_COMMENTS :

   220

        match = regex.match(comment)

   221

   222

        if match :

   224

            break

   225

    else :

   232

   233

    owner = matches.pop('owner', None)

   234

   235

    if owner :

   236

        mail_match = ZONE_OWNER_MAIL.match(owner)

   237

   238

        if mail_match :

   239

            mail_matches = mail_match.groupdict()

   240

   241

            owner = mail_matches['owner']

   242

            yield 'mail', mail_matches['mail'].strip()

   245

   246

    for field, value in matches.iteritems() :

   247

        if value :

   248

            yield field, value.strip()

   249

   250

NONE_OWNERS = set((

   251

    u'tech',

   252

    u'atk',

   253

    u'toimisto',

   254

))

   255

   256

def process_host_owner_ldap (options, host, info) :

   257

    """

   258

        Yield guesses for user from LDAP.

   259

    """

   260

   261

    if info.get('mail') :

   262

        for user in options.ldap.users.filter(

   263

                { 'mailLocalAddress': info['mail'] },

   264

                { 'uid': info['mail'] },

   265

        ) :

   266

            yield user, None

   267

   268

    if info.get('group') and info.get('owner') :

   269

        groups = options.ldap.groups.filter(cn=info['group'])

   270

   271

        for group in groups :

   272

            for user in options.ldap.users.filter({

   273

                'gidNumber': group['gidNumber'],

   274

                'cn': info['owner'],

   275

            }) :

   276

                yield user, group

   277

   278

    if info.get('owner') :

   279

            for user in options.ldap.users.filter({

   280

                'cn': info['owner'],

   281

            }) :

   282

                yield user, None

   283

   284

def process_host_owner (options, host, info) :

   285

    """

   286

        Return (owner, comment) for host based on info, or None.

   287

    """

   288

   292

        return False

   293

   294

    # from ldap?

   295

    for ldap in process_host_owner_ldap(options, host, info) :

   296

        user, group = ldap

   297

   298

        if not group :

   299

            # get group from ldap

   300

            group = options.ldap.users.group(user)

   301

   302

        return user['uid'], u"{group} / {user}".format(

   303

                user    = user.getunicode('cn'),

   304

                group   = group.getunicode('cn'),

   305

        )

   306

   307

def process_host_comments (options, host, info) :

   308

    """

   309

        Process host fields from comment.

   310

   311

        Attempts to find owner from LDAP..

   312

    """

   313

   314

    log.debug("%s: %s", host, info)

   315

   316

    owner = process_host_owner(options, host, info) 

   317

   318

    if owner is False :

   319

        # do not mark any owner

   320

        pass

   321

   322

    elif owner :

   323

        owner, comment = owner

   324

   325

        log.info("%s: %s (%s)", host, owner, comment)

   326

   327

        yield 'comment-owner', comment

   328

        yield 'owner', owner,

   329

   330

    else :

   331

        log.warn("%s: unknown owner: %s", host, info)

   332

        yield 'comment-owner', "{group} / {owner}".format(

   333

                group   = info.get('group', ''),

   334

                owner   = info.get('owner', ''),

   335

        )

   336

   337

    if info.get('host') :

   338

        yield 'comment-host', info['host']

   339

   340

def process_hosts_comments (options, import_hosts) :

   341

    """

   342

        Parse out comments from host imports..

   343

    """

   344

   345

    for host, field, value in import_hosts :

   346

        if field != 'comment':

   347

            yield host, field, value

   348

            continue

   349

   350

        fields = dict(process_zone_comment(options, host, value))

   351

   352

        if options.dump_host_comments :

   353

            print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(

   354

                    host        = host,

   355

                    comment     = value,

   356

                    group       = fields.get('group', ''),

   357

                    owner       = fields.get('owner', ''),

   358

                    mail        = fields.get('mail', ''),

   359

                    hostinfo    = fields.get('host', ''),

   360

            ).encode('utf-8')

   361

   362

   363

        for field, value in process_host_comments(options, host, fields) :

   364

            yield host, field, value

   365

   366

def apply_hosts_import (options) :

   367

    """

   368

        Import host infos from given files.

   369

    """

   370

   371

    if options.import_zone_hosts:

   372

        for info in import_zone_hosts(options,

   373

                pvl.args.apply_file(options.import_zone_hosts, 'r', options.input_charset)) :

   374

            yield info

   375

   376

    if options.import_dhcp_hosts:

   377

        for info in import_dhcp_conf(options,

   378

                pvl.args.apply_file(options.import_dhcp_hosts, 'r', options.input_charset)) :

   379

            yield info

   380

   381

def import_hosts (options) :

   382

    """

   383

        Import hosts from dns/dhcp.

   384

    """

   385

   386

    import_hosts = apply_hosts_import(options)

   387

    import_hosts = process_hosts_comments(options, import_hosts)

   388

   389

    # gather

   390

    hosts = collections.defaultdict(lambda: collections.defaultdict(list))

   391

   392

    for host, field, value in import_hosts :

   393

        hosts[host][field].append(value)

   394

   395

    return hosts.iteritems()

   396

   397

def process_export_hosts (options, hosts) :

   398

    if options.output_prefix :

   399

        prefix = ipaddr.IPNetwork(options.output_prefix)

   400

    else :

   401

        prefix = None

   402

   403

    for host, fields in hosts :

   404

        ip = fields.get('ip')

   405

   406

        # sort by IP

   407

        if ip :

   408

            sort = ip[0]

   409

        else :

   410

            # fake, to sort correctly

   411

            sort = ipaddr.IPAddress(0)

   412

   413

        # select

   414

        if prefix:

   415

            if not (ip and ip in prefix) :

   416

                continue

   417

   418

        yield sort, host, fields

   419

   420

def export_hosts (options, hosts) :

   421

    """

   422

        Export hosts to file.

   423

    """

   424

   425

    file = pvl.args.apply_file(options.output_hosts, 'w', options.output_charset)

   426

   427

    # filter + sort

   428

    hosts = [(host, fields) for sort, host, fields in sorted(process_export_hosts(options, hosts))]

   429

   430

    for host, fields in hosts :

   431

        for comment in fields.get('comment-host', ()):

   432

            print >>file, u"# {comment}".format(comment=comment)

   433

   434

        print >>file, u"[{host}]".format(host=host)

   435

   436

        for field, fmt in (

   437

                ('ip',              None),

   438

                ('ethernet',        None),

   439

                ('owner',           u"\t{field:15} = {value} # {fields[comment-owner][0]}"),

   440

                ('alias',           None),

   441

        ) :

   442

            if not fmt :

   443

                fmt = u"\t{field:15} = {value}"

   444

   445

            values = fields.get(field, ())

   446

   447

            if len(values) > 1 :

   448

                for index, value in enumerate(values, 1) :

   449

                    print >>file, fmt.format(

   450

                            field   = "{field}.{index}".format(field=field, index=index),

   451

                            value   = value,

   452

                            fields  = fields

   453

                    )

   454

            elif len(values) > 0 :

   455

                value, = values

   456

                print >>file, fmt.format(field=field, value=value, fields=fields)

   457

   458

        print >>file

   459

   460

def main (argv) :

   461

    options, args = parse_options(argv)

   462

   463

    options.ldap = pvl.ldap.args.apply(options)

   464

   465

    if args :

   466

        # direct from file

   467

        hosts = pvl.args.apply_files(args, 'r', options.input_charset)

   468

    else :

   469

        # import

   470

        hosts = import_hosts(options)

   471

   472

    # output

   473

    if options.output_hosts :

   474

        export_hosts(options, hosts)

   475

   476

if __name__ == '__main__':

   477

    pvl.args.main(main)